branch

.github/actions/process-replay/action.yml

@@ -11,5 +11,5 @@ runs:
         git fetch origin $CURRENT_SHA
         export COMMIT_MESSAGE=$(git show -s --format=%B "$CURRENT_SHA")
         export CURRENT_HEAD=$(git rev-parse HEAD)
-        cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && IGNORE_OOB=1 PYTHONPATH=. python3 process_replay.py
+        cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
         git checkout $CURRENT_HEAD  # restore to branch

.github/actions/setup-tinygrad/action.yml

@@ -29,10 +29,6 @@ inputs:
     description: "Install CUDA?"
     required: false
     default: 'false'
-  ocelot:
-    description: "Install gpuocelot?"
-    required: false
-    default: 'false'
   webgpu:
     description: "Install webgpu?"
     required: false
@@ -45,19 +41,31 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python ${{ inputs.python-version }}
-      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
 
     # **** Caching packages ****
+    # TODO: key should include input.deps, but it can't since it can't contain commas
 
-    - name: Cache Python packages
-      id: restore-venv
+    - name: Cache Python packages (Linux)
+      if: inputs.key != '' && runner.os == 'Linux'
       uses: actions/cache@v4
       with:
-        path: ${{ github.workspace }}/.venv
-        key: venv-${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-${{ inputs.deps }}-${{ inputs.pydeps }}-${{ hashFiles('**/setup.py') }}-${{ env.PYTHON_CACHE_VERSION }}
+        path: ${{ env.Python3_ROOT_DIR }}/lib/python${{ inputs.python-version }}/site-packages
+        key: python-package-${{ inputs.key }}-${{ hashFiles('**/setup.py') }}
+    - name: Cache Python packages (macOS)
+      if: inputs.key != '' && runner.os == 'macOS'
+      uses: actions/cache@v4
+      with:
+        path: /Users/runner/Library/Python/${{ inputs.python-version }}/lib/python/site-packages
+        key: osx-python-package-${{ inputs.key }}-${{ hashFiles('**/setup.py') }}
+    - name: Cache Python packages (Windows)
+      if: inputs.key != '' && runner.os == 'Windows'
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.Python3_ROOT_DIR }}\Lib\site-packages
+        key: windows-python-package-${{ inputs.key }}-${{ hashFiles('**/setup.py') }}
 
     # **** Caching downloads ****
 
@@ -76,133 +84,43 @@ runs:
 
     # **** Python deps ****
 
-    - name: Install dependencies in venv (with extra)
-      if: inputs.deps != '' && steps.restore-venv.outputs.cache-hit != 'true'
+    - name: Install dependencies (with extra)
+      if: inputs.deps != ''
       shell: bash
-      run: |
-        python -m venv .venv
-        if [[ "$RUNNER_OS" == "Windows" ]]; then
-          source .venv/Scripts/activate
-        else
-          . .venv/bin/activate
-        fi
-        python -m pip install -e ".[${{ inputs.deps }}]" ${{ inputs.pydeps }} --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/
-    - name: Install dependencies in venv (without extra)
-      if: inputs.deps == '' && steps.restore-venv.outputs.cache-hit != 'true'
+      run: pip install ${{ (runner.os == 'macOS' && '--user') || (runner.os != 'macOS' && '') }} -e ".[${{ inputs.deps }}]" ${{ inputs.pydeps }} --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/
+    - name: Install dependencies (without extra)
+      if: inputs.deps == ''
       shell: bash
-      run: |
-        python -m venv .venv
-        if [[ "$RUNNER_OS" == "Windows" ]]; then
-          source .venv/Scripts/activate
-        else
-          . .venv/bin/activate
-        fi
-        python -m pip install -e . ${{ inputs.pydeps }}
-    - name: Set up venv environment
-      shell: bash
-      run: |
-        echo "VIRTUAL_ENV=${{ github.workspace }}/.venv" >> "$GITHUB_ENV"
-        echo "OMP_NUM_THREADS=1" >> "$GITHUB_ENV"
-        # no buffers should be over 300MB in CI
-        echo "MAX_BUFFER_SIZE=300000000" >> "$GITHUB_ENV"
-        if [[ "$RUNNER_OS" == "Windows" ]]; then
-          echo "${{ github.workspace }}/.venv/Scripts" >> "$GITHUB_PATH"
-        else
-          echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH"
-        fi
+      run: pip install ${{ (runner.os == 'macOS' && '--user') || (runner.os != 'macOS' && '') }} -e . ${{ inputs.pydeps }}
 
-    # ******************* apt *******************
-    - name: Setup apt
-      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true')
+    # **** OpenCL ****
+
+    - name: Install OpenCL
+      if: inputs.opencl == 'true'
       shell: bash
       run: |
-        sudo chown -R $USER:$USER /var/cache/apt/archives
-
-        echo 'Acquire::GzipIndexes "true";' | sudo tee /etc/apt/apt.conf.d/gzip
         echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
-        echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | sudo tee -a /etc/apt/apt.conf.d/99keep-debs
-
-    - name: Add OpenCL Repo
-      if: inputs.opencl == 'true' && runner.os == 'Linux'
-      shell: bash
-      run: echo "deb [ allow-insecure=yes ] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
-
-    - name: Add AMD Repo (Linux)
-      if: inputs.amd == 'true' && runner.os == 'Linux'
-      shell: bash
-      run: |
-        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
-        sudo tee /etc/apt/sources.list.d/rocm.list <<EOF
-        deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2 $(lsb_release -cs) main
-        EOF
-        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
-
-    - name: Add LLVM Repo (Linux)
-      if: inputs.llvm == 'true' && runner.os == 'Linux'
-      shell: bash
-      run: |
-        wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-        echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-20 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-
-    - name: Compute Package List + Hash
-      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true')
-      id: apt-pkgs
-      shell: bash
-      run: |
-        pkgs=""
-        # **** OpenCL ****
-        if [[ "${{ inputs.opencl }}" == "true" ]]; then
-          pkgs+=" opencl-headers \
-            intel-oneapi-runtime-openmp=2023.2.1-16 intel-oneapi-runtime-compilers-common=2023.2.1-16 intel-oneapi-runtime-compilers=2023.2.1-16 \
-            intel-oneapi-runtime-dpcpp-sycl-opencl-cpu=2023.2.1-16 intel-oneapi-runtime-tbb-common=2021.10.0-49541 \
-            intel-oneapi-runtime-tbb=2021.10.0-49541 intel-oneapi-runtime-opencl=2023.2.1-16"
-        fi
-        # **** AMD ****
-        if [[ "${{ inputs.amd }}" == "true" ]]; then
-          pkgs+=" hsa-rocr comgr hsa-rocr-dev liburing-dev libibverbs-dev libc6-dev"
-        fi
-        # **** CUDA ****
-        if [[ "${{ inputs.cuda }}" == "true" ]]; then
-          pkgs+=" git g++ cmake ninja-build llvm-15-dev zlib1g-dev libglew-dev \
-            flex bison libfl-dev libboost-thread-dev libboost-filesystem-dev nvidia-cuda-toolkit-gcc libzstd-dev"
-        fi
-        # **** WebGPU (dependencies for software-based vulkan) ****
-        if [[ "${{ inputs.webgpu }}" == "true" ]]; then
-          pkgs+=" libgl1 libglx-mesa0 libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers"
-        fi
-        # **** LLVM ****
-        if [[ "${{ inputs.llvm }}" == "true" ]]; then
-          pkgs+=" libllvm20 clang-20 lld-20"
-        fi
-
-        echo "pkgs=$pkgs" >> "$GITHUB_OUTPUT"
-        echo "hash=$(echo -n "$pkgs" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
-
-    - name: Cache apt
-      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true')
-      uses: actions/cache@v4
-      with:
-        path: /var/cache/apt/archives/
-        key: ${{ runner.os }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.APT_CACHE_VERSION }}
-
-    - name: Run apt Update + Install
-      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true')
-      shell: bash
-      run: |
-        sudo apt -qq update || true
-
-        # ******** do install ********
-        if [[ -n "${{ steps.apt-pkgs.outputs.pkgs }}" ]]; then
-          sudo apt-get -y --allow-unauthenticated --no-install-recommends install ${{ steps.apt-pkgs.outputs.pkgs }}
-        fi
-
-        sudo chown -R $USER:$USER /var/cache/apt/archives/
+        echo "deb [ allow-insecure=yes ] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+        sudo apt update || true
+        sudo apt install --allow-unauthenticated -y --no-install-recommends opencl-headers \
+          intel-oneapi-runtime-openmp=2023.2.1-16 intel-oneapi-runtime-compilers-common=2023.2.1-16 intel-oneapi-runtime-compilers=2023.2.1-16 \
+          intel-oneapi-runtime-dpcpp-sycl-opencl-cpu=2023.2.1-16 intel-oneapi-runtime-tbb-common=2021.10.0-49541 \
+          intel-oneapi-runtime-tbb=2021.10.0-49541 intel-oneapi-runtime-opencl=2023.2.1-16
 
     # **** AMD ****
-    - name: Setup AMD (Linux)
+
+    - name: Install AMD (Linux)
       if: inputs.amd == 'true' && runner.os == 'Linux'
       shell: bash
       run: |
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        sudo tee /etc/apt/sources.list.d/rocm.list <<'EOF'
+        deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.1.2 jammy main
+        EOF
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update || true
+        sudo apt install --no-install-recommends --allow-unauthenticated -y hsa-rocr comgr hsa-rocr-dev liburing-dev libc6-dev
         cargo build --release --manifest-path ./extra/remu/Cargo.toml
         sudo ln -sf ${{ github.workspace }}/extra/remu/target/release/libremu.so /usr/local/lib/libremu.so
         sudo tee --append /etc/ld.so.conf.d/rocm.conf <<'EOF'
@@ -210,7 +128,7 @@ runs:
           /opt/rocm/lib64
         EOF
         sudo ldconfig
-    - name: Setup AMD comgr+remu (macOS)
+    - name: Install AMD comgr+remu (macOS)
       if: inputs.amd == 'true' && runner.os == 'macOS'
       shell: bash
       run: |
@@ -220,32 +138,33 @@ runs:
           sudo xargs curl -L -o /usr/local/lib/libamd_comgr.dylib
         cargo build --release --manifest-path ./extra/remu/Cargo.toml
 
-    # **** gpuocelot ****
+    # **** CUDA ****
 
-    - name: Install gpuocelot dependencies (MacOS)
-      if: inputs.ocelot == 'true' && runner.os == 'macOS'
+    - name: Install cuda packages (Linux)
+      if: inputs.cuda == 'true' && runner.os == 'Linux'
       shell: bash
       run: |
-        pkgs=(cmake ninja llvm@15 zlib glew flex bison [email protected] zstd ncurses)
-        for f in "${pkgs[@]}"; do
-          brew ls --versions "$f" >/dev/null 2>&1 || brew install --quiet "$f"
-        done
-
-        # Fix boost 1.85 for gpuocelot
-        ln -s /opt/homebrew/opt/[email protected] /opt/homebrew/opt/boost || true
-        ln -s /opt/homebrew/opt/boost/lib/libboost_atomic-mt.dylib /opt/homebrew/opt/boost/lib/libboost_atomic.dylib || true
-        ln -s /opt/homebrew/opt/boost/lib/libboost_thread-mt.dylib /opt/homebrew/opt/boost/lib/libboost_thread.dylib || true
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        sudo apt update -y || true
+        sudo apt install -y --no-install-recommends git g++ cmake ninja-build llvm-15-dev zlib1g-dev libglew-dev \
+          flex bison libfl-dev libboost-thread-dev libboost-filesystem-dev nvidia-cuda-toolkit-gcc libzstd-dev
+    - name: Install gpuocelot dependencies (MacOS)
+      if: inputs.cuda == 'true' && runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew update
+        brew install cmake ninja llvm@15 zlib glew flex bison boost zstd ncurses
     - name: Cache gpuocelot
-      if: inputs.ocelot == 'true'
+      if: inputs.cuda == 'true'
       id: cache-build
       uses: actions/cache@v4
       env:
-        cache-name: cache-gpuocelot-build-1
+        cache-name: cache-gpuocelot-build
       with:
         path: ${{ github.workspace }}/gpuocelot/ocelot
-        key: ${{ runner.os }}-gpuocelot-b16039dc940dc6bc4ea0a98380495769ff35ed99-rebuild-${{ env.BUILD_CACHE_VERSION }}
+        key: ${{ runner.os }}-gpuocelot-b16039dc940dc6bc4ea0a98380495769ff35ed99-rebuild-0
     - name: Clone/compile gpuocelot
-      if: inputs.ocelot == 'true' && steps.cache-build.outputs.cache-hit != 'true'
+      if: inputs.cuda == 'true' && steps.cache-build.outputs.cache-hit != 'true'
       shell: bash
       run: |
         git clone --recurse-submodules https://github.com/gpuocelot/gpuocelot.git ${{ github.workspace }}/gpuocelot
@@ -253,20 +172,14 @@ runs:
         git checkout b16039dc940dc6bc4ea0a98380495769ff35ed99
         mkdir build
         cd build
-
-        CMAKE_ARGS="-Wno-dev -G Ninja -DOCELOT_BUILD_TOOLS=OFF -DCMAKE_BUILD_ALWAYS=0 -DBUILD_TESTS_CUDA=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5"
-        if [[ "${{ runner.os }}" == "macOS" ]]; then
-          CMAKE_ARGS="$CMAKE_ARGS -DBoost_INCLUDE_DIR=$(brew --prefix boost)/include -DBoost_LIBRARY_DIR=$(brew --prefix boost)/lib"
-        fi
-
-        cmake .. $CMAKE_ARGS
+        cmake .. -Wno-dev -G Ninja -DOCELOT_BUILD_TOOLS=OFF -DCMAKE_BUILD_ALWAYS=0 -DBUILD_TESTS_CUDA=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5
         ninja
     - name: Install gpuocelot
-      if: inputs.ocelot == 'true'
+      if: inputs.cuda == 'true'
       shell: bash
       run: |
         cd ${{ github.workspace }}/gpuocelot/ocelot/build
-        sudo cp libgpuocelot.${{ runner.os == 'macOS' && 'dylib' || 'so' }} /usr/${{ runner.os == 'macOS' && 'local/' || '' }}lib/
+        sudo cp libgpuocelot.${{ runner.os == 'macOS' && 'dylib' || 'so' }} /usr/${{ runner.os == 'macOS' && 'local/' || ''}}lib/
 
     # **** WebGPU ****
 
@@ -275,7 +188,13 @@ runs:
       shell: bash
       run: |
         sudo curl -L https://github.com/wpmed92/pydawn/releases/download/v0.1.6/libwebgpu_dawn.so -o /usr/local/lib/libwebgpu_dawn.so
-        sudo ldconfig
+    - name: Install dependencies for software-based vulkan
+      if: inputs.webgpu == 'true' && runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt update -y || true
+        sudo apt install -y libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+
     - name: Install WebGPU dawn (macOS)
       if: inputs.webgpu == 'true' && runner.os == 'macOS'
       shell: bash
@@ -285,7 +204,18 @@ runs:
 
     # **** LLVM ****
 
+    - name: Install LLVM (Linux)
+      if: inputs.llvm == 'true' && runner.os == 'Linux'
+      shell: bash
+      run: |
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+        echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+        sudo apt update -y || true
+        sudo apt install -y --no-install-recommends libllvm19 clang-19 lld-19
+
     - name: Install LLVM (macOS)
       if: inputs.llvm == 'true' && runner.os == 'macOS'
       shell: bash
-      run: brew install llvm@20
+      run: |
+        brew install llvm

.github/workflows/autogen.yml

@@ -1,91 +0,0 @@
-name: Autogen
-env:
-  # increment this when downloads substantially change to avoid the internet
-  DOWNLOAD_CACHE_VERSION: '12'
-  PYTHON_CACHE_VERSION: '3'
-  APT_CACHE_VERSION: '1'
-  BUILD_CACHE_VERSION: '1'
-  CAPTURE_PROCESS_REPLAY: 1
-  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PYTHONPATH: ${{ github.workspace }}
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    paths: 
-    - 'tinygrad/runtime/autogen/**/*'
-  workflow_dispatch:
-    paths: 
-    - 'tinygrad/runtime/autogen/**/*'
-
-jobs:
-  autogen:
-    name: Autogen
-    runs-on: ubuntu-24.04
-    timeout-minutes: 15
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Setup Environment
-      uses: ./.github/actions/setup-tinygrad
-      with:
-        opencl: 'true'
-        amd: 'true'
-        cuda: 'true'
-        webgpu: 'true'
-        llvm: 'true'
-    - name: Install autogen support packages
-      run: sudo apt-get install -y --no-install-recommends llvm-14-dev libclang-14-dev
-    - name: Verify OpenCL autogen
-      run: |
-        cp tinygrad/runtime/autogen/opencl.py /tmp/opencl.py.bak
-        ./autogen_stubs.sh opencl
-        diff /tmp/opencl.py.bak tinygrad/runtime/autogen/opencl.py
-    - name: Verify CUDA autogen
-      run: |
-        cp tinygrad/runtime/autogen/cuda.py /tmp/cuda.py.bak
-        cp tinygrad/runtime/autogen/nv_gpu.py /tmp/nv_gpu.py.bak
-        ./autogen_stubs.sh cuda
-        ./autogen_stubs.sh nv
-        diff /tmp/cuda.py.bak tinygrad/runtime/autogen/cuda.py
-        diff /tmp/nv_gpu.py.bak tinygrad/runtime/autogen/nv_gpu.py
-    - name: Verify AMD autogen
-      run: |
-        cp tinygrad/runtime/autogen/hsa.py /tmp/hsa.py.bak
-        cp tinygrad/runtime/autogen/kfd.py /tmp/kfd.py.bak
-        cp tinygrad/runtime/autogen/comgr.py /tmp/comgr.py.bak
-        cp tinygrad/runtime/autogen/amd_gpu.py /tmp/amd_gpu.py.bak
-        cp tinygrad/runtime/autogen/sqtt.py /tmp/sqtt.py.bak
-        ./autogen_stubs.sh hsa
-        ./autogen_stubs.sh kfd
-        ./autogen_stubs.sh comgr
-        ./autogen_stubs.sh amd
-        ./autogen_stubs.sh sqtt
-        diff /tmp/hsa.py.bak tinygrad/runtime/autogen/hsa.py
-        diff /tmp/kfd.py.bak tinygrad/runtime/autogen/kfd.py
-        diff /tmp/comgr.py.bak tinygrad/runtime/autogen/comgr.py
-        diff /tmp/amd_gpu.py.bak tinygrad/runtime/autogen/amd_gpu.py
-        diff /tmp/sqtt.py.bak tinygrad/runtime/autogen/sqtt.py
-    - name: Verify Linux autogen
-      run: |
-        cp tinygrad/runtime/autogen/libc.py /tmp/libc.py.bak
-        cp tinygrad/runtime/autogen/io_uring.py /tmp/io_uring.py.bak
-        cp tinygrad/runtime/autogen/ib.py /tmp/ib.py.bak
-        ./autogen_stubs.sh libc
-        ./autogen_stubs.sh io_uring
-        ./autogen_stubs.sh ib
-        diff /tmp/libc.py.bak tinygrad/runtime/autogen/libc.py
-        diff /tmp/io_uring.py.bak tinygrad/runtime/autogen/io_uring.py
-        diff /tmp/ib.py.bak tinygrad/runtime/autogen/ib.py
-    - name: Verify WebGPU autogen
-      run: |
-        cp tinygrad/runtime/autogen/webgpu.py /tmp/webgpu.py.bak
-        ./autogen_stubs.sh webgpu
-        diff /tmp/webgpu.py.bak tinygrad/runtime/autogen/webgpu.py
-    - name: Verify LLVM autogen
-      run: |
-        cp tinygrad/runtime/autogen/llvm.py /tmp/llvm.py.bak
-        ./autogen_stubs.sh llvm
-        diff /tmp/llvm.py.bak tinygrad/runtime/autogen/llvm.py

.github/workflows/benchmark.yml

@@ -24,14 +24,11 @@ on:
 jobs:
   testmacbenchmark:
     name: Mac Benchmark
-    env:
-      # since sudo is required for usbgpu on macos, move the cache to a new location, as some of the files are owned by root
-      PYTHONPYCACHEPREFIX: /tmp/tiny_python_pycache
     runs-on: [self-hosted, macOS]
-    timeout-minutes: 60
+    timeout-minutes: 20
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
@@ -52,26 +49,23 @@ jobs:
     - name: reset process replay
       run: python3.11 test/external/process_replay/reset.py
     - name: Run Stable Diffusion
-      run: BENCHMARK_LOG=stable_diffusion JIT=1 ASSERT_MIN_STEP_TIME=500 python3.11 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
+      run: JIT=1 python3.11 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
     - name: Run Stable Diffusion without fp16
-      run: BENCHMARK_LOG=stable_diffusion_fp32 JIT=1 ASSERT_MIN_STEP_TIME=700 python3.11 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd_no_fp16.txt
+      run: JIT=1 python3.11 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd_no_fp16.txt
     - name: Run Stable Diffusion v2
-      run: BENCHMARK_LOG=stable_diffusion_v2 JIT=1 ASSERT_MIN_STEP_TIME=1600 python3.11 examples/sdv2.py --fp16 --seed 0 --noshow --timing | tee sdv2.txt
-    # process replay can't capture this, the graph is too large
+      run: JIT=1 python3.11 examples/sdv2.py --fp16 --seed 0 --noshow --timing | tee sdv2.txt
     - name: Run SDXL
-      run: BENCHMARK_LOG=stable_diffusion_xl ASSERT_MIN_STEP_TIME=3000 CAPTURE_PROCESS_REPLAY=0 JIT=1 python3.11 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
+      run: JIT=1 python3.11 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
     - name: Run model inference benchmark
       run: METAL=1 python3.11 test/external/external_model_benchmark.py
     - name: Test speed vs torch
-      run: BIG=2 MPS=1 python3.11 test/speed/external_test_speed_v_torch.py | tee torch_speed.txt
+      run: BIG=2 MPS=1 python3.11 test/test_speed_v_torch.py | tee torch_speed.txt
     - name: Test tensor cores
-      run: METAL=1 python3.11 test/opt/test_tensor_cores.py
+      run: METAL=1 python3.11 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded TestLinearizer.test_tensor_cores_padded_uops
     - name: Test AMX tensor cores
       run: |
-        DEBUG=2 CPU=1 CPU_LLVM=0 AMX=1 python3.11 test/opt/test_tensor_cores.py
-        DEBUG=2 CPU=1 CPU_LLVM=1 AMX=1 python3.11 test/opt/test_tensor_cores.py
-        DEBUG=2 CPU=1 CPU_LLVM=0 AMX=1 python3.11 test/opt/test_gen_float4.py TestFloat4.test_float4_multidim_amx TestFloat4.test_float4_multidim_unaligned_load_amx
-        DEBUG=2 CPU=1 CPU_LLVM=1 AMX=1 python3.11 test/opt/test_gen_float4.py TestFloat4.test_float4_multidim_amx TestFloat4.test_float4_multidim_unaligned_load_amx
+        DEBUG=2 CPU=1 AMX=1 python3.11 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded TestLinearizer.test_tensor_cores_padded_uops
+        DEBUG=2 LLVM=1 AMX=1 python3.11 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded TestLinearizer.test_tensor_cores_padded_uops
     - name: Run Tensor Core GEMM (float)
       run: DEBUG=2 SHOULD_USE_TC=1 python3.11 extra/gemm/simple_matmul.py | tee matmul.txt
     - name: Run Tensor Core GEMM (half)
@@ -82,48 +76,34 @@ jobs:
       run: METAL=1 M_START=6 M_STOP=10 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=6 K_STOP=24 K_STEP=1 TC_OPT=2 DEBUG=2 python3.11 ./extra/gemm/fuzz_matmul.py
     - name: Run LLaMA
       run: |
-        BENCHMARK_LOG=llama_nojit JIT=0 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
-        BENCHMARK_LOG=llama JIT=1 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
+        JIT=0 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
+        JIT=1 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
     - name: Run LLaMA with BEAM
-      run: BENCHMARK_LOG=llama_beam JITBEAM=2 IGNORE_BEAM_CACHE=1 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
+      run: JITBEAM=2 IGNORE_BEAM_CACHE=1 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
     - name: Run quantized LLaMA
       run: |
-        BENCHMARK_LOG=llama_int8 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize int8 | tee llama_int8.txt
-        BENCHMARK_LOG=llama_nf4 python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize nf4 | tee llama_nf4.txt
-    - name: Run quantized LLaMA3
-      run: |
-        BENCHMARK_LOG=llama3_int8 python3.11 examples/llama3.py --size 8B --temperature 0 --benchmark --quantize int8 | tee llama3_int8.txt
-        BENCHMARK_LOG=llama3_nf4 python3.11 examples/llama3.py --size 8B --temperature 0 --benchmark --quantize nf4 | tee llama3_nf4.txt
+        python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize int8 | tee llama_int8.txt
+        python3.11 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize nf4 | tee llama_nf4.txt
     #- name: Run LLaMA 7B on 4 (virtual) GPUs
     #  run: python3.11 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
     - name: Run GPT2
       run: |
-        BENCHMARK_LOG=gpt2_nojit JIT=0 python3.11 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
-        BENCHMARK_LOG=gpt2 JIT=1 ASSERT_MIN_STEP_TIME=8 python3.11 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
+        JIT=0 python3.11 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
+        JIT=1 python3.11 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
     - name: Run GPT2 w HALF
-      run: BENCHMARK_LOG=gpt2_half HALF=1 python3.11 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
+      run: HALF=1 python3.11 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
     - name: Run GPT2 w HALF/BEAM
-      run: BENCHMARK_LOG=gpt2_half_beam HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3.11 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
-    - name: Run OLMoE
-      run: BENCHMARK_LOG=olmoe python3.11 examples/olmoe.py
+      run: HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3.11 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
     - name: Train MNIST
       run: time PYTHONPATH=. TARGET_EVAL_ACC_PCT=96.0 python3.11 examples/beautiful_mnist.py | tee beautiful_mnist.txt
     - name: Run 10 CIFAR training steps
-      run: BENCHMARK_LOG=cifar_10steps JIT=1 ASSERT_MIN_STEP_TIME=330 STEPS=10 python3.11 examples/hlb_cifar10.py | tee train_cifar.txt
+      run: JIT=1 STEPS=10 python3.11 examples/hlb_cifar10.py | tee train_cifar.txt
     - name: Run 10 CIFAR training steps w HALF
-      run: BENCHMARK_LOG=cifar_10steps_half JIT=2 ASSERT_MIN_STEP_TIME=385 STEPS=10 DEFAULT_FLOAT=HALF python3.11 examples/hlb_cifar10.py | tee train_cifar_half.txt
+      run: JIT=2 STEPS=10 DEFAULT_FLOAT=HALF python3.11 examples/hlb_cifar10.py | tee train_cifar_half.txt
     #- name: Run 10 CIFAR training steps w BF16
     #  run: STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3.11 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
     - name: Run 10 CIFAR training steps w winograd
-      run: BENCHMARK_LOG=cifar_10steps_wino JIT=1 ASSERT_MIN_STEP_TIME=150 WINO=1 STEPS=10 python3.11 examples/hlb_cifar10.py | tee train_cifar_wino.txt
-    - name: UsbGPU boot time
-      run: sudo -E PYTHONPATH=. DEBUG=2 AM_RESET=1 AMD=1 AMD_IFACE=USB time python3.11 test/test_tiny.py TestTiny.test_plus
-    - name: UsbGPU tiny tests
-      run: sudo -E PYTHONPATH=. AMD=1 AMD_IFACE=USB python3.11 test/test_tiny.py
-    - name: UsbGPU copy speeds
-      run: sudo -E PYTHONPATH=. AMD=1 AMD_IFACE=USB python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds
-    #- name: UsbGPU openpilot test
-    #  run: sudo -E PYTHONPATH=. AMD=1 AMD_IFACE=USB NOLOCALS=0 IMAGE=0 GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx
+      run: JIT=1 WINO=1 STEPS=10 python3.11 examples/hlb_cifar10.py | tee train_cifar_wino.txt
     - uses: actions/upload-artifact@v4
       with:
         name: Speed (Mac)
@@ -135,8 +115,6 @@ jobs:
           llama_beam.txt
           llama_int8.txt
           llama_nf4.txt
-          llama3_int8.txt
-          llama3_nf4.txt
           llama_four_gpu.txt
           gpt2_unjitted.txt
           gpt2_jitted.txt
@@ -160,10 +138,10 @@ jobs:
   testnvidiabenchmark:
     name: tinybox green Benchmark
     runs-on: [self-hosted, Linux, tinyboxgreen]
-    timeout-minutes: 60
+    timeout-minutes: 30
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
@@ -189,22 +167,22 @@ jobs:
     - name: Run model inference benchmark
       run: NV=1 CAPTURE_PROCESS_REPLAY=0 NOCLANG=1 python3 test/external/external_model_benchmark.py
     - name: Test speed vs torch
-      run: NV=1 CAPTURE_PROCESS_REPLAY=0 HALF=1 BIG=2 TORCHCUDA=1 python3 test/speed/external_test_speed_v_torch.py | tee torch_speed.txt
+      run: NV=1 CAPTURE_PROCESS_REPLAY=0 HALF=1 BIG=2 TORCHCUDA=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
     - name: Test speed vs theoretical
       run: NV=1 IGNORE_BEAM_CACHE=1 BEAM_DEBUG=1 DEBUG=1 python -m pytest -rA test/external/speed_v_theoretical.py --durations=20
     - name: Test benchmark allreduce
       run: NV=1 python test/external/external_benchmark_multitensor_allreduce.py
     - name: Test tensor cores
       run: |
-        NV=1 ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
-        NV=1 NV_PTX=1 ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
+        NV=1 ALLOW_TF32=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded TestLinearizer.test_tensor_cores_padded_uops
+        PTX=1 ALLOW_TF32=1 NV=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded TestLinearizer.test_tensor_cores_padded_uops
     - name: Run Tensor Core GEMM (CUDA)
       run: |
         CUDA=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul.txt
         CUDA=1 SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_bfloat16.txt
         CUDA=1 SHOULD_USE_TC=1 ALLOW_TF32=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py | tee matmul_tf32.txt
     - name: Run Tensor Core GEMM (PTX)
-      run: NV=1 NV_PTX=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_ptx.txt
+      run: NV=1 PTX=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_ptx.txt
     - name: Run Tensor Core GEMM (NV)
       run: NV=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_nv.txt
     - name: Test NV=1
@@ -212,37 +190,37 @@ jobs:
     - name: Test CUDA=1
       run: DEBUG=2 CUDA=1 python -m pytest -rA test/test_tiny.py
     - name: Run Stable Diffusion
-      run: BENCHMARK_LOG=stable_diffusion NV=1 python3 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
+      run: NV=1 python3 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
     - name: Run SDXL
-      run: BENCHMARK_LOG=stable_diffusion_xl ASSERT_MIN_STEP_TIME=2000 CAPTURE_PROCESS_REPLAY=0 NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
+      run: NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
     - name: Run LLaMA
       run: |
-        BENCHMARK_LOG=llama_nojit NV=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
-        BENCHMARK_LOG=llama NV=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
+        NV=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
+        NV=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
     - name: Run LLaMA with BEAM
-      run: BENCHMARK_LOG=llama_beam NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
+      run: NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
     # - name: Run LLaMA 7B on 4 GPUs
     #   run: NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
     # - name: Run LLaMA 7B on 6 GPUs
     #   run: NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama.py --gen 1 --size 7B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_six_gpu.txt
     - name: Run LLaMA-3 8B BEAM
-      run: BENCHMARK_LOG=llama3_beam NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama3.py --size 8B --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_beam.txt
+      run: NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama3.py --size 8B --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_beam.txt
     - name: Run LLaMA-3 8B on 4 GPUs with BEAM
-      run: BENCHMARK_LOG=llama3_beam_4gpu NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_four_gpu.txt
+      run: NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_four_gpu.txt
     # - name: Run LLaMA-3 8B on 6 GPUs
     #   run: NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 6 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_six_gpu.txt
     # - name: Run LLaMA-2 70B
     #   run: NV=1 CAPTURE_PROCESS_REPLAY=0 MAX_CONTEXT=256 python3 examples/llama.py --gen 2 --size 70B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_2_70B.txt
     - name: Run Mixtral 8x7B
-      run: time BENCHMARK_LOG=mixtral NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
+      run: time NV=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
     - name: Run GPT2
       run: |
-        BENCHMARK_LOG=gpt2_nojit NV=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
-        BENCHMARK_LOG=gpt2 NV=1 JIT=1 ASSERT_MIN_STEP_TIME=5 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
+        NV=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
+        NV=1 JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
     - name: Run GPT2 w HALF
-      run: BENCHMARK_LOG=gpt2_half NV=1 HALF=1 ASSERT_MIN_STEP_TIME=5 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
+      run: NV=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
     - name: Run GPT2 w HALF/BEAM
-      run: BENCHMARK_LOG=gpt2_half_beam NV=1 HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
+      run: NV=1 HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
     - uses: actions/upload-artifact@v4
       with:
         name: Speed (NVIDIA)
@@ -274,10 +252,10 @@ jobs:
   testmorenvidiabenchmark:
     name: tinybox green Training Benchmark
     runs-on: [self-hosted, Linux, tinyboxgreen]
-    timeout-minutes: 60
+    timeout-minutes: 20
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
@@ -302,30 +280,30 @@ jobs:
     - name: Fuzz Padded Tensor Core GEMM (NV)
       run: NV=1 M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
     - name: Fuzz Padded Tensor Core GEMM (PTX)
-      run: NV=1 NV_PTX=1 M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
+      run: NV=1 PTX=1 M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
     - name: Train MNIST
       run: time PYTHONPATH=. NV=1 TARGET_EVAL_ACC_PCT=96.0 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt
     - name: Run 10 CIFAR training steps
-      run: BENCHMARK_LOG=cifar_10steps ASSERT_MIN_STEP_TIME=85 NV=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
+      run: NV=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
     - name: Run 10 CIFAR training steps w HALF
-      run: BENCHMARK_LOG=cifar_10steps_half ASSERT_MIN_STEP_TIME=68 NV=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
+      run: NV=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
     - name: Run 10 CIFAR training steps w BF16
-      run: BENCHMARK_LOG=cifar_10steps_bf16 ASSERT_MIN_STEP_TIME=75 NV=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
+      run: NV=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
     - name: Run 10 CIFAR training steps w winograd
-      run: BENCHMARK_LOG=cifar_10steps_half_wino ASSERT_MIN_STEP_TIME=35 NV=1 CAPTURE_PROCESS_REPLAY=0 WINO=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
+      run: NV=1 CAPTURE_PROCESS_REPLAY=0 WINO=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
     - name: Run full CIFAR training w 1 GPU
-      run: time BENCHMARK_LOG=cifar NV=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
+      run: time NV=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
     - name: Run full CIFAR training steps w 6 GPUS
-      run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 NV=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
+      run: time CAPTURE_PROCESS_REPLAY=0 NV=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
     - name: Run MLPerf resnet eval on training data
-      run: time BENCHMARK_LOG=resnet_eval NV=1 MODEL=resnet python3 examples/mlperf/model_eval.py
-    #- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
-    #  run: BENCHMARK_LOG=resnet_10steps NV=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
-    #- name: Run 10 MLPerf ResNet50 training steps (6 gpu)
-    #  run: BENCHMARK_LOG=resnet_10steps_6gpu NV=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
+      run: time NV=1 MODEL=resnet python3 examples/mlperf/model_eval.py
+    - name: Run 10 MLPerf ResNet50 training steps (1 gpu)
+      run: NV=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
+    - name: Run 10 MLPerf ResNet50 training steps (6 gpu)
+      run: NV=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
     - name: Run 10 MLPerf Bert training steps (6 gpu)
       # TODO: remove BERT_LAYERS once scheduler is fast
-      run: BENCHMARK_LOG=bert_10steps_6gpu NV=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee train_bert.txt
+      run: NV=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee train_bert.txt
     - uses: actions/upload-artifact@v4
       with:
         name: Speed (NVIDIA Training)
@@ -346,20 +324,16 @@ jobs:
   testamdbenchmark:
     name: tinybox red Benchmark
     runs-on: [self-hosted, Linux, tinybox]
-    timeout-minutes: 60
+    timeout-minutes: 20
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
-    - name: Remove amdgpu
-      run: sudo rmmod amdgpu || true
-    - name: Cleanup running AM processes
-      run: python extra/amdpci/am_smi.py --pids --kill
-    #- name: Insert amdgpu
-    #  run: sudo modprobe amdgpu
+    - name: Insert amdgpu
+      run: sudo modprobe amdgpu
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -378,12 +352,12 @@ jobs:
         rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
     - name: reset process replay
       run: test/external/process_replay/reset.py
-    #- name: setup perflevel
-    #  run: |
-    #    examples/mlperf/training_submission_v4.1/tinycorp/benchmarks/bert/implementations/tinybox_red/setup.sh
-    #    rocm-smi
-    #- name: Show off tinybox
-    #  run: /opt/rocm/bin/rocm-bandwidth-test
+    - name: setup perflevel
+      run: |
+        examples/mlperf/training_submission_v4.1/tinycorp/benchmarks/bert/implementations/tinybox_red/setup.sh
+        rocm-smi
+    - name: Show off tinybox
+      run: /opt/rocm/bin/rocm-bandwidth-test
     # TODO: unstable on AMD
     #- name: Run model inference benchmark
     #  run: LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 NOCLANG=1 python3 test/external/external_model_benchmark.py
@@ -391,63 +365,62 @@ jobs:
     #- name: Test speed vs torch
     #  run: |
     #    python3 -c "import torch; print(torch.__version__)"
-    #    LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 BIG=2 TORCHCUDA=1 python3 test/speed/external_test_speed_v_torch.py | tee torch_speed.txt
+    #    LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 BIG=2 TORCHCUDA=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
     - name: Test speed vs theoretical
       run: AMD=1 IGNORE_BEAM_CACHE=1 BEAM_DEBUG=1 DEBUG=1 python -m pytest -rA test/external/speed_v_theoretical.py --durations=20
     - name: Test tensor cores
       run: |
-        AMD=1 AMD_LLVM=0 python3 test/opt/test_tensor_cores.py
-        AMD=1 AMD_LLVM=1 python3 test/opt/test_tensor_cores.py
+        AMD=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded_amd TestLinearizer.test_tensor_cores_padded_uops
         AMD=1 SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
     - name: Run Tensor Core GEMM (AMD)
       run: AMD=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py | tee matmul_amd.txt
     - name: Test AMD=1
       run: DEBUG=2 AMD=1 python -m pytest -rA test/test_tiny.py
-    #- name: Test HIP=1
-    #  run: DEBUG=2 HIP=1 python -m pytest -rA test/test_tiny.py
+    - name: Test HIP=1
+      run: DEBUG=2 HIP=1 python -m pytest -rA test/test_tiny.py
     # TODO: AMD compiler bug causes this to fail
     #- name: Fuzz Padded Tensor Core GEMM
     #  run: HSA=1 M_START=12 M_STOP=20 M_STEP=1 N_START=12 N_STOP=20 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 DEBUG=2 python3 ./extra/gemm/fuzz_matmul.py
-    #- name: Remove amdgpu
-    #  run: sleep 10 && sudo rmmod amdgpu # sleep a bit to let the driver unload the prev pid.
+    - name: Remove amdgpu
+      run: sleep 5 && sudo rmmod amdgpu # sleep a bit to let the driver unload the prev pid.
     - name: Test AM cold start time
       run: time AMD=1 AM_RESET=1 python3 test/test_tiny.py TestTiny.test_plus
     - name: Test AM warm start time
       run: time AMD=1 python3 test/test_tiny.py TestTiny.test_plus
     - name: Run Stable Diffusion
-      run: BENCHMARK_LOG=stable_diffusion ASSERT_MIN_STEP_TIME=450 AMD=1 python3 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
+      run: AMD=1 python3 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing | tee sd.txt
     - name: Run SDXL
-      run: BENCHMARK_LOG=stable_diffusion_xl ASSERT_MIN_STEP_TIME=1400 CAPTURE_PROCESS_REPLAY=0 AMD=1 python3 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
+      run: AMD=1 python3 examples/sdxl.py --seed 0 --noshow --timing | tee sdxl.txt
     - name: Run LLaMA 7B
       run: |
-        BENCHMARK_LOG=llama_nojit AMD=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
-        BENCHMARK_LOG=llama AMD=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
+        AMD=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
+        AMD=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
     - name: Run LLaMA 7B with BEAM
-      run: BENCHMARK_LOG=llama_beam AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
+      run: AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
     # - name: Run LLaMA 7B on 4 GPUs
     #   run: AMD=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
     # - name: Run LLaMA 7B on 6 GPUs
     #   run: AMD=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama.py --gen 1 --size 7B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_six_gpu.txt
     - name: Run LLaMA-3 8B BEAM
-      run: BENCHMARK_LOG=llama3_beam AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama3.py --size 8B --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_beam.txt
+      run: AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama3.py --size 8B --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_beam.txt
     - name: Run LLaMA-3 8B on 4 GPUs with BEAM
-      run: BENCHMARK_LOG=llama3_beam_4gpu AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_four_gpu.txt
+      run: AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_four_gpu.txt
     # - name: Run LLaMA-3 8B on 6 GPUs
     #   run: AMD=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 6 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0 | tee llama3_six_gpu.txt
-    #- name: Restore amdgpu
-    #  run: sudo modprobe amdgpu
+    - name: Restore amdgpu
+      run: sudo modprobe amdgpu
     # - name: Run LLaMA-2 70B
     #   run: AMD=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama.py --gen 2 --size 70B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_2_70B.txt
     - name: Run Mixtral 8x7B
-      run: time BENCHMARK_LOG=mixtral AMD=1 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
+      run: time AMD=1 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
     - name: Run GPT2
       run: |
-        BENCHMARK_LOG=gpt2_nojit AMD=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
-        BENCHMARK_LOG=gpt2 AMD=1 JIT=1 ASSERT_MIN_STEP_TIME=5 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
+        AMD=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
+        AMD=1 JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
     - name: Run GPT2 w HALF
-      run: BENCHMARK_LOG=gpt2_half AMD=1 HALF=1 ASSERT_MIN_STEP_TIME=5 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
+      run: AMD=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
     - name: Run GPT2 w HALF/BEAM
-      run: BENCHMARK_LOG=gpt2_half_beam AMD=1 HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
+      run: AMD=1 HALF=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
     - uses: actions/upload-artifact@v4
       with:
         name: Speed (AMD)
@@ -476,18 +449,16 @@ jobs:
   testmoreamdbenchmark:
     name: tinybox red Training Benchmark
     runs-on: [self-hosted, Linux, tinybox]
-    timeout-minutes: 60
+    timeout-minutes: 30
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
     - name: Remove amdgpu
       run: sudo rmmod amdgpu || true
-    - name: Cleanup running AM processes
-      run: python extra/amdpci/am_smi.py --pids --kill
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -508,19 +479,26 @@ jobs:
     - name: Train MNIST
       run: time PYTHONPATH=. AMD=1 TARGET_EVAL_ACC_PCT=96.0 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt
     - name: Run 10 CIFAR training steps
-      run: BENCHMARK_LOG=cifar_10steps ASSERT_MIN_STEP_TIME=85 AMD=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
+      run: AMD=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
     - name: Run 10 CIFAR training steps w HALF
-      run: BENCHMARK_LOG=cifar_10steps_half ASSERT_MIN_STEP_TIME=188 AMD=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
-    # - name: Run 10 CIFAR training steps w BF16
-    #   run: BENCHMARK_LOG=cifar_10steps_bf16 ASSERT_MIN_STEP_TIME=288 AMD=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
+      run: AMD=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
+    - name: Run 10 CIFAR training steps w BF16
+      run: AMD=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
     - name: Run 10 CIFAR training steps w winograd
-      run: BENCHMARK_LOG=cifar_10steps_half_wino ASSERT_MIN_STEP_TIME=66 AMD=1 WINO=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
+      run: AMD=1 WINO=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
     - name: Run full CIFAR training w 1 GPU
-      run: time BENCHMARK_LOG=cifar AMD=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
-    #- name: Run full CIFAR training steps w 6 GPUS
-    #  run: time BENCHMARK_LOG=cifar_6gpu AMD=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
-    #- name: Run full CIFAR training steps w 6 GPUS (REMOTE)
-    #  run: time BENCHMARK_LOG=cifar_6gpu_remote REMOTE=1 REMOTEDEV=AMD DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu_remote.txt
+      run: time AMD=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
+    - name: Run full CIFAR training steps w 6 GPUS
+      run: time AMD=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
+    - name: Run MLPerf resnet eval
+      run: time AMD=1 MODEL=resnet python3 examples/mlperf/model_eval.py
+    - name: Run 10 MLPerf ResNet50 training steps (1 gpu)
+      run: AMD=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
+    - name: Run 10 MLPerf ResNet50 training steps (6 gpu)
+      run: AMD=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
+    - name: Run 10 MLPerf Bert training steps (6 gpu)
+      # TODO: remove BERT_LAYERS once scheduler is fast
+      run: AMD=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee train_bert.txt
     - uses: actions/upload-artifact@v4
       with:
         name: Speed (AMD Training)
@@ -532,55 +510,6 @@ jobs:
           train_cifar_wino.txt
           train_cifar_one_gpu.txt
           train_cifar_six_gpu.txt
-          train_cifar_six_gpu_remote.txt
-    - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
-
-  testmlperfamdbenchmark:
-    name: tinybox red MLPerf Benchmark
-    runs-on: [self-hosted, Linux, tinybox]
-    timeout-minutes: 60
-    defaults:
-      run:
-        shell: bash -e -o pipefail {0}
-    if: github.repository_owner == 'tinygrad'
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Remove amdgpu
-      run: sudo rmmod amdgpu || true
-    - name: Cleanup running AM processes
-      run: python extra/amdpci/am_smi.py --pids --kill
-    - name: Symlink models and datasets
-      run: |
-        mkdir -p weights
-        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
-        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
-        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
-        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
-        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
-        mkdir -p extra/datasets
-        ln -s /raid/datasets/imagenet extra/datasets/imagenet
-    - name: setup staging db
-      if: github.ref == 'refs/heads/update_benchmark_staging'
-      run: |
-        echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
-        rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
-    - name: reset process replay
-      run: test/external/process_replay/reset.py
-    - name: Run MLPerf resnet eval
-      run: time BENCHMARK_LOG=resnet_eval AMD=1 MODEL=resnet python3 examples/mlperf/model_eval.py
-    #- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
-    #  run: BENCHMARK_LOG=resnet_10steps AMD=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
-    #- name: Run 10 MLPerf ResNet50 training steps (6 gpu)
-    #  run: BENCHMARK_LOG=resnet_10steps_6gpu AMD=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
-    - name: Run 10 MLPerf Bert training steps (6 gpu)
-      # TODO: remove BERT_LAYERS once scheduler is fast
-      run: BENCHMARK_LOG=bert_10steps_6gpu AMD=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee train_bert.txt
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Speed (AMD MLPerf)
-        path: |
           train_resnet.txt
           train_resnet_one_gpu.txt
           train_bert.txt
@@ -593,7 +522,7 @@ jobs:
     timeout-minutes: 20
     defaults:
       run:
-        shell: bash -e -o pipefail {0}
+        shell: bash -o pipefail {0}
     if: github.repository_owner == 'tinygrad'
     steps:
     - name: Checkout Code
@@ -605,30 +534,22 @@ jobs:
         rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
     - name: reset process replay
       run: test/external/process_replay/reset.py
-    - name: benchmark openpilot 0.9.9 driving_vision
-      run: BENCHMARK_LOG=openpilot_0_9_9_vision ASSERT_MIN_STEP_TIME=30 PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/driving_vision.onnx
-    - name: benchmark openpilot 0.9.9 driving_policy
-      run: BENCHMARK_LOG=openpilot_0_9_9_policy ASSERT_MIN_STEP_TIME=45 PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/driving_policy.onnx
-    - name: benchmark openpilot 0.9.9 dmonitoring
-      run: BENCHMARK_LOG=openpilot_0_9_9_dmonitoring ASSERT_MIN_STEP_TIME=70 PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/dmonitoring_model.onnx
-    - name: openpilot compile3 0.9.9 driving_vision
-      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/driving_vision.onnx
-    - name: openpilot compile3 0.9.9 driving_policy
-      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/driving_policy.onnx
-    - name: openpilot compile3 0.9.9 dmonitoring
-      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.9.9/selfdrive/modeld/models/dmonitoring_model.onnx
-    - name: openpilot compile3 Space Lab policy + vision
-      run: |
-        PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/22aec22a10ce09384d4a4af2a0bbff08d54af7e0c888503508f356fae4ff0e29
-        PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/c824f68646a3b94f117f01c70dc8316fb466e05fbd42ccdba440b8a8dc86914b
-    - name: benchmark MobileNetV2 on DSP
-      run: |
-        # generate quantized weights
-        ln -s /data/home/tiny/tinygrad/extra/datasets/imagenet extra/datasets/imagenet
-        ln -s /data/home/tiny/tinygrad/testsig-*.so .
-        PYTHONPATH=. CC=clang-19 CPU=1 CPU_LLVM=0 QUANT=1 CNT=0 python3 examples/test_onnx_imagenet.py https://github.com/xamcat/mobcat-samples/raw/refs/heads/master/onnx_runtime/InferencingSample/InferencingSample/mobilenetv2-7.onnx /tmp/model.quant.onnx
-        # benchmark on DSP with NOOPT=1, the devectorizer has issues
-        PYTHONPATH=. CC=clang-19 DSP=1 DONT_REALIZE_EXPAND=1 NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx
+    - name: validate openpilot 0.9.7
+      run: PYTHONPATH=. FLOAT16=0 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.7/selfdrive/modeld/models/supercombo.onnx | tee openpilot_image_0_9_7.txt
+    - name: benchmark openpilot 0.9.4
+      run: PYTHONPATH=. QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.4/selfdrive/modeld/models/supercombo.onnx | tee openpilot_0_9_4.txt
+    - name: benchmark openpilot 0.9.7
+      run: PYTHONPATH=. QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.7/selfdrive/modeld/models/supercombo.onnx | tee openpilot_0_9_7.txt
+    - name: benchmark openpilot w IMAGE=2 0.9.4
+      run: PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.4/selfdrive/modeld/models/supercombo.onnx | tee openpilot_image_0_9_4.txt
+    - name: benchmark openpilot w IMAGE=2 0.9.7
+      run: PYTHONPATH=. NOLOCALS=1 FLOAT16=1 IMAGE=2 QCOM=1 taskset -c 4-7 python3 test/external/external_benchmark_openpilot.py https://github.com/commaai/openpilot/raw/v0.9.7/selfdrive/modeld/models/supercombo.onnx | tee openpilot_image_0_9_7.txt
+    - name: openpilot compile3 0.9.7
+      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.9.7/selfdrive/modeld/models/supercombo.onnx
+    - name: openpilot compile3 0.9.7+ tomb raider
+      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/e8bea2c78ffa92685ece511e9b554122aaf1a79d/selfdrive/modeld/models/supercombo.onnx
+    - name: openpilot dmonitoring compile3 0.9.7
+      run: PYTHONPATH="." QCOM=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.9.7/selfdrive/modeld/models/dmonitoring_model.onnx
     - name: Run process replay tests
       run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
     - uses: actions/upload-artifact@v4
@@ -641,136 +562,3 @@ jobs:
           openpilot_0_9_7.txt
           openpilot_image_0_9_4.txt
           openpilot_image_0_9_7.txt
-
-  testreddriverbenchmark:
-    name: AM Benchmark
-    runs-on: [self-hosted, Linux, tinyboxrandom]
-    timeout-minutes: 20
-    defaults:
-      run:
-        shell: bash -e -o pipefail {0}
-    if: github.repository_owner == 'tinygrad'
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Remove amd modules
-      run: ./extra/hcq/hcq_smi.py amd rmmod
-    - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py amd kill_pids
-    - name: Symlink models and datasets
-      run: |
-        mkdir -p weights
-        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
-        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
-        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
-        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
-        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
-        mkdir -p extra/datasets
-        ln -s /raid/datasets/imagenet extra/datasets/imagenet
-    - name: setup staging db
-      if: github.ref == 'refs/heads/update_benchmark_staging'
-      run: |
-        echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
-        rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
-    - name: reset process replay
-      run: test/external/process_replay/reset.py
-    - name: Test driver cold start time
-      run: time DEBUG=3 AMD=1 AM_RESET=1 python3 test/test_tiny.py TestTiny.test_plus
-    - name: Test driver warm start time
-      run: time DEBUG=3 AMD=1 python3 test/test_tiny.py TestTiny.test_plus
-    # Fails on 9070
-    # - name: Test tensor cores
-    #   run: |
-    #     AMD=1 AMD_LLVM=0 python3 test/test_linearizer.py test/opt/test_tensor_cores.py
-    #     AMD=1 AMD_LLVM=1 python3 test/test_linearizer.py test/opt/test_tensor_cores.py
-    #     AMD=1 SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
-    - name: Run Tensor Core GEMM (AMD)
-      run: AMD=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py | tee am_matmul_amd.txt
-    - name: Test AMD=1
-      run: DEBUG=2 AMD=1 python -m pytest -rA test/test_tiny.py
-    - name: Test DISK copy time
-      run: AMD=1 TESTFILE=/raid/downloads/llama3-8b-sfr/model-00001-of-00004.safetensors python3 test/external/external_benchmark_disk_raw.py
-    - name: Test CPU copy time
-      run: |
-        AMD=1 GRAPH_ONE_KERNEL=1 PYTHONPATH=. NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit
-        AMD=1 GRAPH_ONE_KERNEL=1 PYTHONPATH=. NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit
-    - name: Run full CIFAR training w 1 GPU
-      run: time BENCHMARK_LOG=cifar AMD=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee am_train_cifar_one_gpu.txt
-    # TODO: enable
-    # - name: Run 10 MLPerf ResNet50 training steps (1 gpu)
-    #   run: BENCHMARK_LOG=resnet_10steps AMD=1 MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee am_train_resnet_one_gpu.txt
-    - name: Run 10 MLPerf Bert training steps (1 gpu)
-      # TODO: remove BERT_LAYERS once scheduler is fast
-      run: BENCHMARK_LOG=bert_10steps AMD=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=1 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee am_train_bert_one_gpu.txt
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Speed (AM Driver)
-        path: |
-          am_matmul_amd.txt
-          am_train_cifar_one_gpu.txt
-          am_train_resnet_one_gpu.txt
-          am_train_bert_one_gpu.txt
-    - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py
-
-  testgreendriverbenchmark:
-    name: NV Benchmark
-    runs-on: [self-hosted, Linux, tinyboxrandom]
-    timeout-minutes: 20
-    defaults:
-      run:
-        shell: bash -e -o pipefail {0}
-    if: github.repository_owner == 'tinygrad'
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Remove nv modules
-      run: ./extra/hcq/hcq_smi.py nv rmmod
-    - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py nv kill_pids
-    - name: Symlink models and datasets
-      run: |
-        mkdir -p weights
-        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
-        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
-        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
-        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
-        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
-        mkdir -p extra/datasets
-        ln -s /raid/datasets/imagenet extra/datasets/imagenet
-    - name: setup staging db
-      if: github.ref == 'refs/heads/update_benchmark_staging'
-      run: |
-        echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
-        rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
-    - name: reset process replay
-      run: test/external/process_replay/reset.py
-    - name: Test driver start time
-      run: time DEBUG=3 NV=1 python3 test/test_tiny.py TestTiny.test_plus
-    - name: Test tensor cores
-      run: NV=1 ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
-    - name: Test DISK copy time
-      run: NV=1 TESTFILE=/raid/downloads/llama3-8b-sfr/model-00001-of-00004.safetensors python3 test/external/external_benchmark_disk_raw.py
-    - name: Test CPU copy time
-      run: |
-        NV=1 GRAPH_ONE_KERNEL=1 PYTHONPATH=. NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit
-        NV=1 GRAPH_ONE_KERNEL=1 PYTHONPATH=. NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit
-    - name: Test LLAMA-3
-      run: BENCHMARK_LOG=llama3_beam NV=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 examples/llama3.py --size 8B --benchmark --temperature 0 | tee nv_llama3_beam.txt
-    - name: Run full CIFAR training w 1 GPU
-      run: time BENCHMARK_LOG=cifar NV=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee nv_train_cifar_one_gpu.txt
-    #- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
-    #  run: BENCHMARK_LOG=resnet_10steps NV=1 MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee nv_train_resnet_one_gpu.txt
-    - name: Run 10 MLPerf Bert training steps (1 gpu)
-      # TODO: remove BERT_LAYERS once scheduler is fast
-      run: BENCHMARK_LOG=bert_10steps NV=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=1 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py | tee nv_train_bert_one_gpu.txt
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Speed (NV Driver)
-        path: |
-          nv_llama3_beam.txt
-          nv_train_cifar_one_gpu.txt
-          nv_train_resnet_one_gpu.txt
-          nv_train_bert_one_gpu.txt
-    - name: Run process replay tests
-      run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py

.github/workflows/benchmark_search.yml

@@ -1,34 +0,0 @@
-name: Benchmark with kernel search
-
-on:
-  push:
-    branches:
-      - update_benchmark_search
-  workflow_dispatch:
-
-jobs:
-  run_script_job:
-    runs-on: [self-hosted, Linux, tinybox]
-    if: github.repository_owner == 'tinygrad'
-    timeout-minutes: 100
-
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Remove amdgpu
-      run: sudo rmmod amdgpu || true
-    - name: Cleanup running AM processes
-      run: python extra/amdpci/am_smi.py --pids --kill
-    - name: Run SDXL with new search
-      # TODO: GCVM_L2_PROTECTION_FAULT_STATUS with llvm19
-      run: |
-        BENCHMARK_LOG=search_sdxl PYTHONPATH=. AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 DISABLE_COMPILER_CACHE=1 python examples/sdxl.py --noshow --timing --seed 0
-    - name: Run SDXL with cached search
-      run: |
-        BENCHMARK_LOG=search_sdxl_cached PYTHONPATH=. AMD=1 JITBEAM=2 python examples/sdxl.py --noshow --timing --seed 0
-    - name: Run winograd cifar with new search
-      run: |
-        BENCHMARK_LOG=search_wino_cifar WINO=1 DEFAULT_FLOAT=HALF JITBEAM=4 IGNORE_BEAM_CACHE=1 DISABLE_COMPILER_CACHE=1 BS=1024 STEPS=500 python examples/hlb_cifar10.py
-    - name: Run winograd cifar with cached search
-      run: |
-        BENCHMARK_LOG=search_wino_cifar_cached WINO=1 DEFAULT_FLOAT=HALF JITBEAM=4 BS=1024 STEPS=500 python examples/hlb_cifar10.py

.github/workflows/mlperf.yml

@@ -1,30 +0,0 @@
-name: Run MLPerf Training
-
-on:
-  schedule:
-    - cron: '5 8 * * *'  # Runs at 08:05 UTC (12:05 AM Pacific Time)
-  push:
-    branches:
-      - update_mlperf
-  workflow_dispatch:
-
-jobs:
-  run_script_job:
-    runs-on: [self-hosted, Linux, tinybox]
-    if: github.repository_owner == 'tinygrad'
-    timeout-minutes: 360
-
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    - name: Cleanup running AM processes
-      run: python extra/amdpci/am_smi.py --pids --kill
-    - name: Symlink datasets
-      run: |
-        mkdir -p extra/datasets
-        ln -s /raid/datasets/imagenet extra/datasets/imagenet
-    - name: Run resnet
-      run: |
-        rm "~/.cache/tinygrad/cache_mlperf.db" || true
-        BENCHMARK_LOG=mlpert_train_resnet LOGMLPERF=0 CACHEDB="~/.cache/tinygrad/cache_mlperf.db" examples/mlperf/training_submission_v5.1/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh
-        rm "~/.cache/tinygrad/cache_mlperf.db"

.gitignore

@@ -11,6 +11,7 @@ notebooks
 *.txt
 build
 !examples/tinychat/assets/cdn.jsdelivr.net/npm/[email protected]/build/
+!examples/mlperf/training_submission_*/**/*.txt
 /dist
 *.egg-info
 /env
@@ -61,4 +62,3 @@ site/
 profile_stats
 *.log
 target
-.mypy_cache

.pre-commit-config.yaml

@@ -20,9 +20,15 @@ repos:
         language: system
         always_run: true
         pass_filenames: false
+      - id: devicetests
+        name: select GPU tests
+        entry: env GPU=1 PYTHONPATH="." python3 -m pytest test/test_uops.py test/test_search.py
+        language: system
+        always_run: true
+        pass_filenames: false
       - id: tests
         name: subset of tests
-        entry: env PYTHONPATH="." python3 -m pytest -n=4 test/test_ops.py test/test_dtype.py test/test_schedule.py test/test_assign.py
+        entry: env PYTHONPATH="." python3 -m pytest -n=4 test/unit/ test/test_ops.py test/test_dtype.py test/test_schedule.py test/test_assign.py test/test_symbolic_shapetracker.py
         language: system
         always_run: true
         pass_filenames: false

.pylintrc

@@ -7,7 +7,7 @@ extension-pkg-whitelist=scipy,cereal.messaging.messaging_pyx,PyQt5,av
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,autogen,msm_kgsl.py,runtime,.venv
+ignore=CVS,autogen,msm_kgsl.py,runtime
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
@@ -54,12 +54,11 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=C,R,W0613,W0511,W0212,W0201,W0106,W0603,W0621,W0703,W1201,W1203,E1136,W1514,E1101,W0221,W0105,E0401,abstract-method,W0707
+disable=C,R,W0613,W0511,W0212,W0201,W0106,W0603,W0621,W0703,W1201,W1203,E1136,W1514,E1101,W0221,W0105,E0401,abstract-method
 # E1101 for function binding
 # W0221 for Function class
 # W0105 for comment strings
 # E0401 for missing imports
-# W0707 for not reraising
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

AGENTS.md

@@ -1,17 +0,0 @@
-# tinygrad agents
-
-Hello agent. You are one of the most talented programmers of your generation.
-
-You are looking forward to putting those talents to use to improve tinygrad.
-
-## philosophy
-
-tinygrad is a **tensor** library focused on beauty and minimalism, while still matching the functionality of PyTorch and JAX.
-
-Every line must earn its keep. Prefer readability over cleverness. We believe that if carefully designed, 10 lines can have the impact of 1000.
-
-Never mix functionality changes with whitespace changes. All functionality changes must be tested.
-
-## style
-
-Use **2-space indentation**, and keep lines to a maximum of **150 characters**. Match the existing style.

README.md

@@ -21,11 +21,11 @@ tinygrad: For something between [PyTorch](https://github.com/pytorch/pytorch) an
 
 ---
 
-Despite tinygrad's size, it is a fully featured deep learning framework.
+This may not be the best deep learning framework, but it is a deep learning framework.
 
-Due to its extreme simplicity, it is the easiest framework to add new accelerators to, with support for both inference and training. If XLA is CISC, tinygrad is RISC.
+Due to its extreme simplicity, it aims to be the easiest framework to add new accelerators to, with support for both inference and training. If XLA is CISC, tinygrad is RISC.
 
-tinygrad is now beta software, we [raised some money](https://geohot.github.io/blog/jekyll/update/2023/05/24/the-tiny-corp-raised-5M.html) to make it good. Someday, we will tape out chips.
+tinygrad is still alpha software, but we [raised some money](https://geohot.github.io/blog/jekyll/update/2023/05/24/the-tiny-corp-raised-5M.html) to make it good. Someday, we will tape out chips.
 
 ## Features
 
@@ -39,8 +39,9 @@ Try a matmul. See how, despite the style, it is fused into one kernel with the p
 
 ```sh
 DEBUG=3 python3 -c "from tinygrad import Tensor;
-N = 1024; a, b = Tensor.empty(N, N), Tensor.empty(N, N);
-(a.reshape(N, 1, N) * b.T.reshape(1, N, N)).sum(axis=2).realize()"
+N = 1024; a, b = Tensor.rand(N, N), Tensor.rand(N, N);
+c = (a.reshape(N, 1, N) * b.T.reshape(1, N, N)).sum(axis=2);
+print((c.numpy() - (a.numpy() @ b.numpy())).mean())"
 ```
 
 And we can change `DEBUG` to `4` to see the generated code.
@@ -79,8 +80,9 @@ See [examples/beautiful_mnist.py](examples/beautiful_mnist.py) for the full vers
 
 tinygrad already supports numerous accelerators, including:
 
-- [x] [OpenCL](tinygrad/runtime/ops_cl.py)
-- [x] [CPU](tinygrad/runtime/ops_cpu.py)
+- [x] [GPU (OpenCL)](tinygrad/runtime/ops_gpu.py)
+- [x] [CPU (C Code)](tinygrad/runtime/ops_cpu.py)
+- [x] [LLVM](tinygrad/runtime/ops_llvm.py)
 - [x] [METAL](tinygrad/runtime/ops_metal.py)
 - [x] [CUDA](tinygrad/runtime/ops_cuda.py)
 - [x] [AMD](tinygrad/runtime/ops_amd.py)

autogen_stubs.sh

@@ -9,7 +9,7 @@ if [[ ! $(clang2py -V) ]]; then
   pip install clang==14.0.6
   git clone https://github.com/nimlgen/ctypeslib.git
   cd ctypeslib
-  pip install .
+  pip install --user .
   clang2py -V
   popd
 fi
@@ -35,7 +35,7 @@ def _try_dlopen_$name():
   for candidate in PATHS_TO_TRY:
     try: return ctypes.CDLL(candidate)
     except OSError: pass
-  return None
+  raise RuntimeError("library $name not found")
 EOF
 }
 
@@ -83,12 +83,11 @@ generate_kfd() {
   sed -i "/import functools/a from tinygrad.runtime.support.hcq import FileIOInterface" $BASE/kfd.py
   sed -i "s/def _do_ioctl(__idir, __base, __nr, __user_struct, __fd, \*\*kwargs):/def _do_ioctl(__idir, __base, __nr, __user_struct, __fd:FileIOInterface, \*\*kwargs):/g" $BASE/kfd.py
   sed -i "s/fcntl.ioctl(__fd, (__idir<<30)/__fd.ioctl((__idir<<30)/g" $BASE/kfd.py
-  sed -i "s/!!/not not /g" $BASE/kfd.py
   python3 -c "import tinygrad.runtime.autogen.kfd"
 }
 
 generate_cuda() {
-  clang2py /usr/include/cuda.h --clang-args="-D__CUDA_API_VERSION_INTERNAL" -o $BASE/cuda.py -l /usr/lib/x86_64-linux-gnu/libcuda.so
+  clang2py /usr/include/cuda.h -o $BASE/cuda.py -l /usr/lib/x86_64-linux-gnu/libcuda.so
   sed -i "s\import ctypes\import ctypes, ctypes.util\g" $BASE/cuda.py
   sed -i "s\ctypes.CDLL('/usr/lib/x86_64-linux-gnu/libcuda.so')\ctypes.CDLL(ctypes.util.find_library('cuda'))\g" $BASE/cuda.py
   fixup $BASE/cuda.py
@@ -105,10 +104,10 @@ generate_nvrtc() {
 }
 
 generate_nv() {
-  NVKERN_COMMIT_HASH=81fe4fb417c8ac3b9bdcc1d56827d116743892a5
+  NVKERN_COMMIT_HASH=d6b75a34094b0f56c2ccadf14e5d0bd515ed1ab6
   NVKERN_SRC=/tmp/open-gpu-kernel-modules-$NVKERN_COMMIT_HASH
   if [ ! -d "$NVKERN_SRC" ]; then
-    git clone https://github.com/NVIDIA/open-gpu-kernel-modules $NVKERN_SRC
+    git clone https://github.com/tinygrad/open-gpu-kernel-modules $NVKERN_SRC
     pushd .
     cd $NVKERN_SRC
     git reset --hard $NVKERN_COMMIT_HASH
@@ -117,21 +116,15 @@ generate_nv() {
 
   clang2py -k cdefstum \
     extra/nv_gpu_driver/clc6c0qmd.h \
-    extra/nv_gpu_driver/clcec0qmd.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl0000.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl0080.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl2080.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl2080_notification.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc86f.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc96f.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc761.h \
+    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
+    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc56f.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/class/cl83de.h \
     $NVKERN_SRC/src/nvidia/generated/g_allclasses.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clc6c0.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/class/clcdc0.h \
     $NVKERN_SRC/kernel-open/nvidia-uvm/clc6b5.h \
-    $NVKERN_SRC/kernel-open/nvidia-uvm/clc9b5.h \
     $NVKERN_SRC/kernel-open/nvidia-uvm/uvm_ioctl.h \
     $NVKERN_SRC/kernel-open/nvidia-uvm/uvm_linux_ioctl.h \
     $NVKERN_SRC/kernel-open/nvidia-uvm/hwref/ampere/ga100/dev_fault.h \
@@ -149,7 +142,6 @@ generate_nv() {
     $NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl/ctrlc36f.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl/ctrlcb33.h \
     $NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl/ctrla06c.h \
-    $NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl/ctrl90f1.h \
     --clang-args="-include $NVKERN_SRC/src/common/sdk/nvidia/inc/nvtypes.h -I$NVKERN_SRC/src/common/inc -I$NVKERN_SRC/kernel-open/nvidia-uvm -I$NVKERN_SRC/kernel-open/common/inc -I$NVKERN_SRC/src/common/sdk/nvidia/inc -I$NVKERN_SRC/src/nvidia/arch/nvalloc/unix/include -I$NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl" \
     -o $BASE/nv_gpu.py
   fixup $BASE/nv_gpu.py
@@ -157,8 +149,6 @@ generate_nv() {
   sed -i "s\import ctypes\import ctypes, os\g" $BASE/nv_gpu.py
   sed -i 's/#\?\s\([A-Za-z0-9_]\+\) = MW ( \([0-9]\+\) : \([0-9]\+\) )/\1 = (\2 , \3)/' $BASE/nv_gpu.py # NVC6C0_QMDV03_00 processing
   sed -i 's/#\sdef NVC6C0_QMD\([A-Za-z0-9_()]\+\):/def NVC6C0_QMD\1:/' $BASE/nv_gpu.py
-  sed -i 's/#\sdef NVCEC0_QMD\([A-Za-z0-9_()]\+\):/def NVCEC0_QMD\1:/' $BASE/nv_gpu.py
-  sed -E -i -n '/^def (NVCEC0_QMDV05_00_RELEASE)(_ENABLE)\(i\):/{p;s//\1'"0"'\2=\1\2(0)\n\1'"1"'\2=\1\2(1)/;H;b};p;${x;s/^\n//;p}' "$BASE/nv_gpu.py"
   sed -i 's/#\s*return MW(\([0-9i()*+]\+\):\([0-9i()*+]\+\))/    return (\1 , \2)/' $BASE/nv_gpu.py
   sed -i 's/#\?\s*\(.*\)\s*=\s*\(NV\)\?BIT\(32\)\?\s*(\s*\([0-9]\+\)\s*)/\1 = (1 << \4)/' $BASE/nv_gpu.py # name = BIT(x) -> name = (1 << x)
   sed -i "s/UVM_\([A-Za-z0-9_]\+\) = \['i', '(', '\([0-9]\+\)', ')'\]/UVM_\1 = \2/" $BASE/nv_gpu.py # UVM_name = ['i', '(', '<num>', ')'] -> UVM_name = <num>
@@ -167,30 +157,8 @@ generate_nv() {
   sed -n '1i\
 nv_status_codes = {}
 /^NV_STATUS_CODE/ { s/^NV_STATUS_CODE(\([^,]*\), *\([^,]*\), *"\([^"]*\)") *.*$/\1 = \2\nnv_status_codes[\1] = "\3"/; p }' $NVKERN_SRC/src/common/sdk/nvidia/inc/nvstatuscodes.h >> $BASE/nv_gpu.py
+
   python3 -c "import tinygrad.runtime.autogen.nv_gpu"
-
-  clang2py -k cdefstum \
-    $NVKERN_SRC/src/nvidia/inc/kernel/gpu/fsp/kern_fsp_cot_payload.h \
-    $NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc/gsp/gspifpub.h \
-    $NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h \
-    $NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h \
-    $NVKERN_SRC/src/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h \
-    $NVKERN_SRC/src/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h \
-    $NVKERN_SRC/src/common/uproc/os/common/include/libos_init_args.h \
-    $NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h \
-    $NVKERN_SRC/src/common/shared/msgq/inc/msgq/msgq_priv.h \
-    $NVKERN_SRC/src/nvidia/inc/kernel/vgpu/rpc_headers.h \
-    $NVKERN_SRC/src/nvidia/inc/kernel/vgpu/rpc_global_enums.h \
-    $NVKERN_SRC/src/nvidia/generated/g_rpc-structures.h \
-    $NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc/fsp/fsp_nvdm_format.h \
-    extra/nv_gpu_driver/g_rpc-message-header.h \
-    extra/nv_gpu_driver/gsp_static_config.h \
-    extra/nv_gpu_driver/vbios.h \
-    --clang-args="-DRPC_MESSAGE_STRUCTURES -DRPC_STRUCTURES -include $NVKERN_SRC/src/common/sdk/nvidia/inc/nvtypes.h -I$NVKERN_SRC/src/nvidia/generated -I$NVKERN_SRC/src/common/inc -I$NVKERN_SRC/src/nvidia/inc -I$NVKERN_SRC/src/nvidia/interface/ -I$NVKERN_SRC/src/nvidia/inc/kernel -I$NVKERN_SRC/src/nvidia/inc/libraries -I$NVKERN_SRC/src/nvidia/arch/nvalloc/common/inc -I$NVKERN_SRC/kernel-open/nvidia-uvm -I$NVKERN_SRC/kernel-open/common/inc -I$NVKERN_SRC/src/common/sdk/nvidia/inc -I$NVKERN_SRC/src/nvidia/arch/nvalloc/unix/include -I$NVKERN_SRC/src/common/sdk/nvidia/inc/ctrl" \
-    -o $BASE/nv/nv.py
-
-  fixup $BASE/nv/nv.py
-  python3 -c "import tinygrad.runtime.autogen.nv.nv"
 }
 
 generate_amd() {
@@ -198,7 +166,11 @@ generate_amd() {
   clang2py -k cdefstum \
     extra/hip_gpu_driver/sdma_registers.h \
     extra/hip_gpu_driver/nvd.h \
+    extra/hip_gpu_driver/kfd_pm4_headers_ai.h \
+    extra/hip_gpu_driver/soc21_enum.h \
+    extra/hip_gpu_driver/sdma_v6_0_0_pkt_open.h \
     extra/hip_gpu_driver/gc_11_0_0_offset.h \
+    extra/hip_gpu_driver/gc_10_3_0_offset.h \
     extra/hip_gpu_driver/sienna_cichlid_ip_offset.h \
     --clang-args="-I/opt/rocm/include -x c++" \
     -o $BASE/amd_gpu.py
@@ -236,21 +208,6 @@ generate_io_uring() {
   fixup $BASE/io_uring.py
 }
 
-generate_ib() {
-  clang2py -k cdefstum \
-    /usr/include/infiniband/verbs.h \
-    /usr/include/infiniband/verbs_api.h \
-    /usr/include/infiniband/ib_user_ioctl_verbs.h \
-    /usr/include/rdma/ib_user_verbs.h \
-    -o $BASE/ib.py
-
-  sed -i "s\import ctypes\import ctypes, ctypes.util\g" "$BASE/ib.py"
-  sed -i "s\FIXME_STUB\libibverbs\g" "$BASE/ib.py"
-  sed -i "s\FunctionFactoryStub()\ctypes.CDLL(ctypes.util.find_library('ibverbs'), use_errno=True)\g" "$BASE/ib.py"
-
-  fixup $BASE/ib.py
-}
-
 generate_libc() {
   clang2py -k cdefstum \
     $(dpkg -L libc6-dev | grep sys/mman.h) \
@@ -263,7 +220,7 @@ generate_libc() {
 
   sed -i "s\import ctypes\import ctypes, ctypes.util, os\g" $BASE/libc.py
   sed -i "s\FIXME_STUB\libc\g" $BASE/libc.py
-  sed -i "s\FunctionFactoryStub()\None if (libc_path := ctypes.util.find_library('c')) is None else ctypes.CDLL(libc_path, use_errno=True)\g" $BASE/libc.py
+  sed -i "s\FunctionFactoryStub()\None if (libc_path := ctypes.util.find_library('c')) is None else ctypes.CDLL(libc_path)\g" $BASE/libc.py
 
   fixup $BASE/libc.py
 }
@@ -372,6 +329,70 @@ generate_am() {
     -o $BASE/am/pm4_nv.py
   fixup $BASE/am/pm4_nv.py
 
+  clang2py -k cdefstum \
+    $AMKERN_INC/vega10_enum.h \
+    -o $BASE/am/vega10.py
+  fixup $BASE/am/vega10.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/navi10_enum.h \
+    -o $BASE/am/navi10.py
+  fixup $BASE/am/navi10.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/soc21_enum.h \
+    -o $BASE/am/soc21.py
+  fixup $BASE/am/soc21.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/soc24_enum.h \
+    -o $BASE/am/soc24.py
+  fixup $BASE/am/soc24.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mp/mp_13_0_0_offset.h \
+    $AMKERN_INC/asic_reg/mp/mp_13_0_0_sh_mask.h \
+    -o $BASE/am/mp_13_0_0.py
+  fixup $BASE/am/mp_13_0_0.py
+
+  # 14_0_3 reuses 14_0_2
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mp/mp_14_0_2_offset.h \
+    $AMKERN_INC/asic_reg/mp/mp_14_0_2_sh_mask.h \
+    -o $BASE/am/mp_14_0_3.py
+  fixup $BASE/am/mp_14_0_3.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mp/mp_11_0_offset.h \
+    $AMKERN_INC/asic_reg/mp/mp_11_0_sh_mask.h \
+    -o $BASE/am/mp_11_0.py
+  fixup $BASE/am/mp_11_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/gc/gc_9_4_3_offset.h \
+    $AMKERN_INC/asic_reg/gc/gc_9_4_3_sh_mask.h \
+    extra/amdpci/overlay/gc_9_4_3.h \
+    -o $BASE/am/gc_9_4_3.py
+  fixup $BASE/am/gc_9_4_3.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/gc/gc_10_3_0_offset.h \
+    $AMKERN_INC/asic_reg/gc/gc_10_3_0_sh_mask.h \
+    -o $BASE/am/gc_10_3_0.py
+  fixup $BASE/am/gc_10_3_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/gc/gc_11_0_0_offset.h \
+    $AMKERN_INC/asic_reg/gc/gc_11_0_0_sh_mask.h \
+    -o $BASE/am/gc_11_0_0.py
+  fixup $BASE/am/gc_11_0_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/gc/gc_12_0_0_offset.h \
+    $AMKERN_INC/asic_reg/gc/gc_12_0_0_sh_mask.h \
+    -o $BASE/am/gc_12_0_0.py
+  fixup $BASE/am/gc_12_0_0.py
+
   clang2py -k cdefstum \
     extra/hip_gpu_driver/sdma_registers.h \
     $AMKERN_AMD/amdgpu/vega10_sdma_pkt_open.h \
@@ -393,6 +414,72 @@ generate_am() {
     -o $BASE/am/sdma_6_0_0.py
   fixup $BASE/am/sdma_6_0_0.py
 
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_3_0_0_offset.h \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_3_0_0_sh_mask.h \
+    -o $BASE/am/mmhub_3_0_0.py
+  fixup $BASE/am/mmhub_3_0_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_3_0_2_offset.h \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_3_0_2_sh_mask.h \
+    -o $BASE/am/mmhub_3_0_2.py
+  fixup $BASE/am/mmhub_3_0_2.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbio/nbio_2_3_offset.h \
+    $AMKERN_INC/asic_reg/nbio/nbio_2_3_sh_mask.h \
+    -o $BASE/am/nbio_2_3_0.py
+  fixup $BASE/am/nbio_2_3_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_2_0_offset.h \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_2_0_sh_mask.h \
+    -o $BASE/am/nbio_7_2_0.py
+  fixup $BASE/am/nbio_7_2_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_4_1_0_offset.h \
+    $AMKERN_INC/asic_reg/mmhub/mmhub_4_1_0_sh_mask.h \
+    -o $BASE/am/mmhub_4_1_0.py
+  fixup $BASE/am/mmhub_4_1_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbio/nbio_4_3_0_offset.h \
+    $AMKERN_INC/asic_reg/nbio/nbio_4_3_0_sh_mask.h \
+    -o $BASE/am/nbio_4_3_0.py
+  fixup $BASE/am/nbio_4_3_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbif/nbif_6_3_1_offset.h \
+    $AMKERN_INC/asic_reg/nbif/nbif_6_3_1_sh_mask.h \
+    -o $BASE/am/nbif_6_3_1.py
+  fixup $BASE/am/nbif_6_3_1.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_9_0_offset.h \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_9_0_sh_mask.h \
+    -o $BASE/am/nbio_7_9_0.py
+  fixup $BASE/am/nbio_7_9_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_11_0_offset.h \
+    $AMKERN_INC/asic_reg/nbio/nbio_7_11_0_sh_mask.h \
+    -o $BASE/am/nbio_7_11_0.py
+  fixup $BASE/am/nbio_7_11_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/oss/osssys_6_0_0_offset.h \
+    $AMKERN_INC/asic_reg/oss/osssys_6_0_0_sh_mask.h \
+    -o $BASE/am/osssys_6_0_0.py
+  fixup $BASE/am/osssys_6_0_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/oss/osssys_7_0_0_offset.h \
+    $AMKERN_INC/asic_reg/oss/osssys_7_0_0_sh_mask.h \
+    -o $BASE/am/osssys_7_0_0.py
+  fixup $BASE/am/osssys_7_0_0.py
+
   clang2py -k cdefstum \
     $AMKERN_AMD/pm/swsmu/inc/pmfw_if/smu_v13_0_0_ppsmc.h \
     $AMKERN_AMD/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h \
@@ -406,8 +493,20 @@ generate_am() {
     $AMKERN_AMD/pm/swsmu/inc/pmfw_if/smu14_driver_if_v14_0.h \
     extra/amdpci/headers/amdgpu_smu.h \
     --clang-args="-include stdint.h" \
-    -o $BASE/am/smu_v14_0_2.py
-  fixup $BASE/am/smu_v14_0_2.py
+    -o $BASE/am/smu_v14_0_3.py
+  fixup $BASE/am/smu_v14_0_3.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/hdp/hdp_6_0_0_offset.h \
+    $AMKERN_INC/asic_reg/hdp/hdp_6_0_0_sh_mask.h \
+    -o $BASE/am/hdp_6_0_0.py
+  fixup $BASE/am/hdp_6_0_0.py
+
+  clang2py -k cdefstum \
+    $AMKERN_INC/asic_reg/hdp/hdp_7_0_0_offset.h \
+    $AMKERN_INC/asic_reg/hdp/hdp_7_0_0_sh_mask.h \
+    -o $BASE/am/hdp_7_0_0.py
+  fixup $BASE/am/hdp_7_0_0.py
 }
 
 generate_sqtt() {
@@ -429,19 +528,6 @@ generate_webgpu() {
   python3 -c "import tinygrad.runtime.autogen.webgpu"
 }
 
-generate_libusb() {
-  clang2py -k cdefstum \
-    /usr/include/libusb-1.0/libusb.h \
-    -o $BASE/libusb.py
-
-  fixup $BASE/libusb.py
-  sed -i "s\import ctypes\import ctypes, ctypes.util, os\g" $BASE/libusb.py
-  sed -i "s/FIXME_STUB/libusb/g" "$BASE/libusb.py"
-  sed -i "s/libusb_le16_to_cpu = libusb_cpu_to_le16//g" "$BASE/libusb.py"
-  sed -i "s/FunctionFactoryStub()/None if (lib_path:=os.getenv('LIBUSB_PATH', ctypes.util.find_library('usb-1.0'))) is None else ctypes.CDLL(lib_path)/g" "$BASE/libusb.py"
-  python3 -c "import tinygrad.runtime.autogen.libusb"
-}
-
 if [ "$1" == "opencl" ]; then generate_opencl
 elif [ "$1" == "hip" ]; then generate_hip
 elif [ "$1" == "comgr" ]; then generate_comgr
@@ -452,11 +538,9 @@ elif [ "$1" == "kfd" ]; then generate_kfd
 elif [ "$1" == "nv" ]; then generate_nv
 elif [ "$1" == "amd" ]; then generate_amd
 elif [ "$1" == "am" ]; then generate_am
-elif [ "$1" == "nvdrv" ]; then generate_nvdrv
 elif [ "$1" == "sqtt" ]; then generate_sqtt
 elif [ "$1" == "qcom" ]; then generate_qcom
 elif [ "$1" == "io_uring" ]; then generate_io_uring
-elif [ "$1" == "ib" ]; then generate_ib
 elif [ "$1" == "libc" ]; then generate_libc
 elif [ "$1" == "llvm" ]; then generate_llvm
 elif [ "$1" == "kgsl" ]; then generate_kgsl
@@ -464,7 +548,6 @@ elif [ "$1" == "adreno" ]; then generate_adreno
 elif [ "$1" == "pci" ]; then generate_pci
 elif [ "$1" == "vfio" ]; then generate_vfio
 elif [ "$1" == "webgpu" ]; then generate_webgpu
-elif [ "$1" == "libusb" ]; then generate_libusb
 elif [ "$1" == "all" ]; then generate_opencl; generate_hip; generate_comgr; generate_cuda; generate_nvrtc; generate_hsa; generate_kfd; generate_nv; generate_amd; generate_io_uring; generate_libc; generate_am; generate_webgpu
 else echo "usage: $0 <type>"
 fi

docs/abstractions2.py

@@ -7,30 +7,28 @@
 
 print("******** first, the runtime ***********")
 
-from tinygrad.runtime.ops_cpu import ClangJITCompiler, CPUDevice, CPUProgram
-
-cpu = CPUDevice()
+from tinygrad.runtime.ops_cpu import ClangJITCompiler, MallocAllocator, CPUProgram
 
 # allocate some buffers
-out = cpu.allocator.alloc(4)
-a = cpu.allocator.alloc(4)
-b = cpu.allocator.alloc(4)
+out = MallocAllocator.alloc(4)
+a = MallocAllocator.alloc(4)
+b = MallocAllocator.alloc(4)
 
 # load in some values (little endian)
-cpu.allocator._copyin(a, memoryview(bytearray([2,0,0,0])))
-cpu.allocator._copyin(b, memoryview(bytearray([3,0,0,0])))
+MallocAllocator._copyin(a, memoryview(bytearray([2,0,0,0])))
+MallocAllocator._copyin(b, memoryview(bytearray([3,0,0,0])))
 
 # compile a program to a binary
 lib = ClangJITCompiler().compile("void add(int *out, int *a, int *b) { out[0] = a[0] + b[0]; }")
 
 # create a runtime for the program
-fxn = cpu.runtime("add", lib)
+fxn = CPUProgram("add", lib)
 
 # run the program
 fxn(out, a, b)
 
 # check the data out
-print(val := cpu.allocator._as_buffer(out).cast("I").tolist()[0])
+print(val := MallocAllocator._as_buffer(out).cast("I").tolist()[0])
 assert val == 5
 
 
@@ -41,31 +39,31 @@ DEVICE = "CPU"   # NOTE: you can change this!
 import struct
 from tinygrad.dtype import dtypes
 from tinygrad.device import Buffer, Device
-from tinygrad.uop.ops import UOp, Ops
+from tinygrad.ops import UOp, Ops
 from tinygrad.shape.shapetracker import ShapeTracker
 
 # allocate some buffers + load in values
 out = Buffer(DEVICE, 1, dtypes.int32).allocate()
 a = Buffer(DEVICE, 1, dtypes.int32).allocate().copyin(memoryview(bytearray(struct.pack("I", 2))))
 b = Buffer(DEVICE, 1, dtypes.int32).allocate().copyin(memoryview(bytearray(struct.pack("I", 3))))
-# NOTE: a._buf is the same as the return from cpu.allocator.alloc
+# NOTE: a._buf is the same as the return from MallocAllocator.alloc
 
 # describe the computation
 buf_1 = UOp(Ops.DEFINE_GLOBAL, dtypes.int32.ptr(), (), 1)
 buf_2 = UOp(Ops.DEFINE_GLOBAL, dtypes.int32.ptr(), (), 2)
-ld_1 = UOp(Ops.LOAD, dtypes.int32, (buf_1.view(ShapeTracker.from_shape((1,))),))
-ld_2 = UOp(Ops.LOAD, dtypes.int32, (buf_2.view(ShapeTracker.from_shape((1,))),))
+ld_1 = UOp(Ops.LOAD, dtypes.int32, (buf_1, ShapeTracker.from_shape((1,)).to_uop()))
+ld_2 = UOp(Ops.LOAD, dtypes.int32, (buf_2, ShapeTracker.from_shape((1,)).to_uop()))
 alu = ld_1 + ld_2
 output_buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int32.ptr(), (), 0)
-st_0 = UOp(Ops.STORE, dtypes.void, (output_buf.view(ShapeTracker.from_shape((1,))), alu))
+st_0 = UOp(Ops.STORE, dtypes.void, (output_buf, ShapeTracker.from_shape((1,)).to_uop(), alu))
 s = UOp(Ops.SINK, dtypes.void, (st_0,))
 
 # convert the computation to a "linearized" format (print the format)
-from tinygrad.engine.realize import get_program, CompiledRunner
-program = get_program(s, Device[DEVICE].renderer)
+from tinygrad.engine.realize import get_kernel, CompiledRunner
+kernel = get_kernel(Device[DEVICE].renderer, s).linearize()
 
 # compile a program (and print the source)
-fxn = CompiledRunner(program)
+fxn = CompiledRunner(kernel.to_program())
 print(fxn.p.src)
 # NOTE: fxn.clprg is the CPUProgram
 
@@ -80,9 +78,7 @@ print("******** third, the UOp ***********")
 
 from tinygrad.engine.realize import run_schedule
 from tinygrad.engine.schedule import create_schedule_with_vars
-from tinygrad.helpers import RANGEIFY
-from tinygrad.schedule.kernelize import get_kernelize_map
-from tinygrad.schedule.rangeify import get_rangeify_map
+from tinygrad.engine.grouper import get_becomes_map
 
 # allocate some values + load in values
 a = UOp.new_buffer(DEVICE, 1, dtypes.int32)
@@ -95,10 +91,10 @@ out = a + b
 s = UOp(Ops.SINK, dtypes.void, (out,))
 
 # group the computation into kernels
-becomes_map = get_rangeify_map(s) if RANGEIFY else get_kernelize_map(s)
+becomes_map = get_becomes_map(s)
 
 # the compute maps to an assign
-assign = becomes_map[a+b].base
+assign = becomes_map[a+b]
 
 # the first source is the output buffer (data)
 assert assign.src[0].op is Ops.BUFFER
@@ -107,7 +103,7 @@ assert assign.src[1].op is Ops.KERNEL
 
 # schedule the kernel graph in a linear list
 s = UOp(Ops.SINK, dtypes.void, (assign,))
-sched, _ = create_schedule_with_vars(s)
+sched, _, becomes_map = create_schedule_with_vars(s)
 assert len(sched) == 1
 
 # DEBUGGING: print the compute ast
@@ -115,7 +111,7 @@ print(sched[-1].ast)
 # NOTE: sched[-1].ast is the same as st_0 above
 
 # the output will be stored in a new buffer
-out = assign.buf_uop
+out = becomes_map[assign]
 assert out.op is Ops.BUFFER and not out.buffer.is_allocated()
 print(out)
 

docs/abstractions3.py

@@ -36,7 +36,7 @@ optim.schedule_step()   # this will step the optimizer without running realize
 # 3. Create a schedule.
 
 # The weight Tensors have been assigned to, but not yet realized. Everything is still lazy at this point
-# l1.uop and l2.uop define a computation graph
+# l1.lazydata and l2.lazydata define a computation graph
 
 from tinygrad.engine.schedule import ScheduleItem
 schedule: List[ScheduleItem] = Tensor.schedule(l1, l2)

docs/developer/hcq.md

@@ -52,7 +52,7 @@ Signals are device-dependent structures used for synchronization and timing in H
 The following Python code demonstrates the usage of signals:
 
 ```python
-signal = your_device.new_signal(value=0)
+signal = your_device.signal_t()
 
 HWQueue().timestamp(signal) \
          .signal(signal, value_to_fire) \

docs/developer/kernelize.md

@@ -34,7 +34,7 @@ print(out) # <Tensor <UOp METAL (1,) int (<Ops.ASSIGN: 66>, None)> on METAL with
 The multiply Tensor stays the same because it is fused. The output Tensor's UOp becomes a new ASSIGN UOp:
 
 ```py
-print(out.uop)
+print(out.lazydata)
 ```
 
 The first source is the output BUFFER:
@@ -72,7 +72,7 @@ Once a Tensor is kernelized, all children will LOAD its BUFFER, instead of fusin
 ```py
 child = out+2
 child.kernelize()
-print(child.uop.src[1].arg.ast)
+print(child.lazydata.src[1].arg.ast)
 ```
 
 ```

docs/developer/layout.md

@@ -1,60 +0,0 @@
-# tinygrad directory layout
-
-This explains the flow of a big graph down to programs.
-
-Directories are listed in order of how they are processed.
-
----
-
-## tinygrad/schedule
-
-Group UOps into kernels.
-
-::: tinygrad.schedule.kernelize.get_kernelize_map
-    options:
-        members: false
-        show_labels: false
-        show_source: false
-
----
-
-## tinygrad/codegen/opt
-
-Transforms the ast into an optimized ast. This is where BEAM search and heuristics live.
-
----
-
-## tinygrad/codegen
-
-Transform the optimized ast into a linearized list of UOps.
-
-::: tinygrad.codegen.full_rewrite
-    options:
-        members: false
-        show_labels: false
-        show_source: false
-
----
-
-## tinygrad/renderer
-
-Transform the linearized list of UOps into a program, represented as a string.
-
-::: tinygrad.renderer.Renderer
-    options:
-        members:
-            - render
-        show_labels: false
-        show_source: false
-
----
-
-## tinygrad/engine
-
-Abstracted high level interface to the runtimes.
-
-::: tinygrad.engine.realize.get_program
-    options:
-        members: false
-        show_labels: false
-        show_source: false

docs/developer/speed.md

@@ -68,4 +68,4 @@ We have a simple framework in tinygrad for adding these ALU blocks and achieving
 
 ### Indexing
 
-Indexing determines the address of the memory we need to load. GPUs often have less integer math resources than floating point math, so this can sometimes be the bottleneck. We have a symbolic math engine in our rewrite rules to simplify indexing before it's emitted to the kernel. Newer NVIDIA GPUs have a "Tensor Memory Accelerator" to assist with fast indexing, however, this is not supported in tinygrad yet.
+Indexing determines the address of the memory we need to load. GPUs often have less integer math resources than floating point math, so this can sometimes be the bottleneck. We have a symbolic math engine in our rewrite rules to simplifiy indexing before it's emitted to the kernel. Newer NVIDIA GPUs have a "Tensor Memory Accelerator" to assist with fast indexing, however, this is not supported in tinygrad yet.

docs/developer/uop.md

@@ -1,10 +1,10 @@
-::: tinygrad.uop.ops.UOp
+::: tinygrad.ops.UOp
     options:
         members: false
         members_order: source
         show_labels: false
 
-::: tinygrad.uop.ops.Ops
+::: tinygrad.ops.Ops
     options:
         members: true
         members_order: source

docs/env_vars.md

@@ -3,7 +3,7 @@
 This is a list of environment variable that control the runtime behavior of tinygrad and its examples.
 Most of these are self-explanatory, and are usually used to set an option at runtime.
 
-Example: `CL=1 DEBUG=4 python3 -m pytest`
+Example: `GPU=1 DEBUG=4 python3 -m pytest`
 
 However you can also decorate a function to set a value only inside that function.
 
@@ -31,16 +31,20 @@ These control the behavior of core tinygrad even when used as a library.
 Variable | Possible Value(s) | Description
 ---|---|---
 DEBUG               | [1-7]      | enable debugging output (operations, timings, speed, generated code and more)
-CL                  | [1]        | enable OpenCL backend
+GPU                 | [1]        | enable the GPU (OpenCL) backend
 CUDA                | [1]        | enable CUDA backend
 AMD                 | [1]        | enable AMD backend
 NV                  | [1]        | enable NV backend
 METAL               | [1]        | enable Metal backend (for Mac M1 and after)
-CPU                 | [1]        | enable CPU backend
+METAL_XCODE         | [1]        | enable Metal using macOS Xcode SDK
+CPU                 | [1]        | enable CPU (Clang) backend
+LLVM                | [1]        | enable LLVM backend
 BEAM                | [#]        | number of beams in kernel beam search
 DEFAULT_FLOAT       | [HALF, ...]| specify the default float dtype (FLOAT32, HALF, BFLOAT16, FLOAT64, ...), default to FLOAT32
 IMAGE               | [1-2]      | enable 2d specific optimizations
 FLOAT16             | [1]        | use float16 for images instead of float32
+PTX                 | [1]        | enable the specialized [PTX](https://docs.nvidia.com/cuda/parallel-thread-execution/) assembler for Nvidia GPUs. If not set, defaults to generic CUDA codegen backend.
+PROFILE             | [1]        | enable profiling. This feature is supported in NV, AMD, QCOM and METAL backends.
 VISIBLE_DEVICES     | [list[int]]| restricts the NV/AMD devices that are available. The format is a comma-separated list of identifiers (indexing starts with 0).
 JIT                 | [0-2]      | 0=disabled, 1=[jit enabled](quickstart.md#jit) (default), 2=jit enabled, but graphs are disabled
 VIZ                 | [1]        | 0=disabled, 1=[viz enabled](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/viz)

docs/index.md

@@ -42,7 +42,7 @@ There's nothing special about a "Module" class in tinygrad, it's just a normal c
 
 ### tinygrad is functional
 
-In tinygrad, you can do [`x.conv2d(w, b)`](tensor/ops.md/#tinygrad.Tensor.conv2d) or [`x.sparse_categorical_crossentropy(y)`](tensor/ops.md/#tinygrad.Tensor.sparse_categorical_crossentropy). We do also have a [`Conv2D`](nn.md/#tinygrad.nn.Conv2d) class like PyTorch if you want a place to keep the state, but all stateless operations don't have classes.
+In tinygrad, you can do [`x.conv2d(w, b)`](tensor/ops.md/#tinygrad.Tensor.conv2d) or [`x.sparse_categorical_cross_entropy(y)`](tensor/ops.md/#tinygrad.Tensor.sparse_categorical_crossentropy). We do also have a [`Conv2D`](nn.md/#tinygrad.nn.Conv2d) class like PyTorch if you want a place to keep the state, but all stateless operations don't have classes.
 
 ### tinygrad is lazy
 

docs/ramp.py

@@ -1,293 +0,0 @@
-#!/usr/bin/env python3
-
-# this file is a "ramp" for people new to tinygrad to think about how to approach it
-# it is runnable and editable.
-# whenever you see stuff like DEBUG=2 or CPU=1 discussed, these are environment variables
-# in a unix shell like bash `DEBUG=2 CPU=1 python docs/ramp.py`
-
-# this pip installs tinygrad master for the system
-# the -e allows you to edit the tinygrad folder and update system tinygrad
-# tinygrad is pure Python, so you are encouraged to do this
-# git pull in the tinygrad directory will also get you the latest
-"""
-git clone https://github.com/tinygrad/tinygrad.git
-cd tinygrad
-python3 -m pip install -e .
-"""
-
-# %% ********
-print("******* PART 1 *******")
-
-# we start with a Device.
-# a Device is where Tensors are stored and compute is run
-# tinygrad autodetects the best device on your system and makes it the DEFAULT
-from tinygrad import Device
-print(Device.DEFAULT)  # on Mac, you can see this prints METAL
-
-# now, lets create a Tensor
-from tinygrad import Tensor, dtypes
-t = Tensor([1,2,3,4])
-
-# you can see this Tensor is on the DEFAULT device with int dtype and shape (4,)
-assert t.device == Device.DEFAULT
-assert t.dtype == dtypes.int
-assert t.shape == (4,)
-
-# unlike in torch, if we print it, it doesn't print the contents
-# this is because tinygrad is lazy
-# this Tensor has not been computed yet
-print(t)
-# <Tensor <UOp METAL (4,) int (<Ops.COPY: 7>, None)> on METAL with grad None>
-
-# the ".uop" property on Tensor contains the specification of how to compute it
-print(t.uop)
-"""
-UOp(Ops.COPY, dtypes.int, arg=None, src=(
-  UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-    UOp(Ops.UNIQUE, dtypes.void, arg=0, src=()),
-    UOp(Ops.DEVICE, dtypes.void, arg='PYTHON', src=()),)),
-  UOp(Ops.DEVICE, dtypes.void, arg='METAL', src=()),))
-"""
-# as you can see, it's specifying a copy from PYTHON device
-# which is where the [1,2,3,4] array lives
-
-# UOps are the specification language in tinygrad
-# they are immutable and form a DAG
-# they have a "Ops", a "dtype", a tuple of srcs (parents), and an arg
-
-t.realize()
-# if we want to "realize" a tensor, we can with the "realize" method
-# now when we look at the uop, it's changed
-print(t.uop)
-"""
-UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-  UOp(Ops.UNIQUE, dtypes.void, arg=1, src=()),
-  UOp(Ops.DEVICE, dtypes.void, arg='METAL', src=()),))
-"""
-# the copy was actually run, and now the "uop" of the Tensor is just a BUFFER
-# if you run this script with DEBUG=2 in the environment, you can see the copy happen
-# *** METAL      1 copy       16,   METAL <- PYTHON ...
-
-# now let's do some compute
-# we look at the uop to see the specification of the compute
-t_times_2 = t * 2
-print(t_times_2.uop)
-"""
-UOp(Ops.MUL, dtypes.int, arg=None, src=(
-  UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-    UOp(Ops.UNIQUE, dtypes.void, arg=1, src=()),
-    x2:=UOp(Ops.DEVICE, dtypes.void, arg='METAL', src=()),)),
-  UOp(Ops.EXPAND, dtypes.int, arg=(4,), src=(
-    UOp(Ops.RESHAPE, dtypes.int, arg=(1,), src=(
-      UOp(Ops.CONST, dtypes.int, arg=2, src=(
-        UOp(Ops.VIEW, dtypes.void, arg=ShapeTracker(views=(View(shape=(), strides=(), offset=0, mask=None, contiguous=True),)), src=(
-           x2,)),)),)),)),))
-"""
-# the BUFFER from above is being multiplied by a CONST 2
-# it's RESHAPEd and EXPANDed to broadcast the CONST to the BUFFER
-
-# we can check the result with
-assert t_times_2.tolist() == [2, 4, 6, 8]
-
-# UOps are both immutable and globally unique
-# if i multiply the Tensor by 4 twice, these result Tensors will have the same uop specification
-t_times_4_try_1 = t * 4
-t_times_4_try_2 = t * 4
-assert t_times_4_try_1.uop is t_times_4_try_2.uop
-# the specification isn't just the same, it's the exact same Python object
-assert t_times_4_try_1 is not t_times_4_try_2
-# the Tensor is a different Python object
-
-# if we realize `t_times_4_try_1` ...
-t_times_4_try_1.realize()
-print(t_times_4_try_2.uop)
-"""
-UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-  UOp(Ops.UNIQUE, dtypes.void, arg=4, src=()),
-  UOp(Ops.DEVICE, dtypes.void, arg='METAL', src=()),))
-"""
-# ... `t_times_4_try_2` also becomes the same BUFFER
-assert t_times_4_try_1.uop is t_times_4_try_2.uop
-# so this print doesn't require any computation, just a copy back to the CPU so we can print it
-print("** only the copy start")
-print(t_times_4_try_2.tolist())  # [4, 8, 12, 16]
-print("** only the copy end")
-# you can confirm this with DEBUG=2, seeing what's printed in between the "**" prints
-
-# tinygrad has an auto differentiation engine that operates according to these same principles
-# the derivative of "log(x)" is "1/x", and you can see this on line 20 of gradient.py
-t_float = Tensor([3.0])
-t_log = t_float.log()
-t_log_grad, = t_log.sum().gradient(t_float)
-# due to how log is implemented, this gradient contains a lot of UOps
-print(t_log_grad.uop)
-# ...not shown here...
-# but if you run with DEBUG=4 (CPU=1 used here for simpler code), you can see the generated code
-"""
-void E_(float* restrict data0, float* restrict data1) {
-  float val0 = *(data1+0);
-  *(data0+0) = (1/val0);
-}
-"""
-# the derivative is close to 1/3
-assert (t_log_grad.item() - 1/3) < 1e-6
-
-# %% ********
-print("******* PART 2 *******")
-
-# we redefine the same t here so this cell can run on it's own
-from tinygrad import Tensor
-t = Tensor([1,2,3,4])
-
-# what's above gives you enough of an understanding to go use tinygrad as a library
-# however, a lot of the beauty of tinygrad is in how easy it is to interact with the internals
-# NOTE: the APIs here are subject to change
-
-t_plus_3_plus_4 = t + 3 + 4
-print(t_plus_3_plus_4.uop)
-"""
-UOp(Ops.ADD, dtypes.int, arg=None, src=(
-  UOp(Ops.ADD, dtypes.int, arg=None, src=(
-    UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-      UOp(Ops.UNIQUE, dtypes.void, arg=1, src=()),
-      x3:=UOp(Ops.DEVICE, dtypes.void, arg='CPU', src=()),)),
-    UOp(Ops.EXPAND, dtypes.int, arg=(4,), src=(
-      UOp(Ops.RESHAPE, dtypes.int, arg=(1,), src=(
-        UOp(Ops.CONST, dtypes.int, arg=3, src=(
-          x7:=UOp(Ops.VIEW, dtypes.void, arg=ShapeTracker(views=(View(shape=(), strides=(), offset=0, mask=None, contiguous=True),)), src=(
-             x3,)),)),)),)),)),
-  UOp(Ops.EXPAND, dtypes.int, arg=(4,), src=(
-    UOp(Ops.RESHAPE, dtypes.int, arg=(1,), src=(
-      UOp(Ops.CONST, dtypes.int, arg=4, src=(
-         x7,)),)),)),))
-"""
-# you can see it's adding both 3 and 4
-
-# but by the time we are actually running the code, it's adding 7
-# `kernelize` will simplify and group the operations in the graph into kernels
-t_plus_3_plus_4.kernelize()
-print(t_plus_3_plus_4.uop)
-"""
-UOp(Ops.ASSIGN, dtypes.int, arg=None, src=(
-  x0:=UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-    UOp(Ops.UNIQUE, dtypes.void, arg=7, src=()),
-    x2:=UOp(Ops.DEVICE, dtypes.void, arg='CPU', src=()),)),
-  UOp(Ops.KERNEL, dtypes.void, arg=<Kernel 12 SINK(<Ops.STORE: 48>,) (__add__,)>, src=(
-     x0,
-    UOp(Ops.BUFFER, dtypes.int, arg=4, src=(
-      UOp(Ops.UNIQUE, dtypes.void, arg=1, src=()),
-       x2,)),)),))
-"""
-# ASSIGN has two srcs, src[0] is the BUFFER that's assigned to, and src[1] is the thing to assign
-# src[1] is the GPU Kernel that's going to be run
-# we can get the ast of the Kernel as follows
-kernel_ast = t_plus_3_plus_4.uop.src[1].arg.ast
-
-# almost everything in tinygrad functions as a rewrite of the UOps
-# the codegen rewrites the ast to a simplified form ready for "rendering"
-from tinygrad.codegen import full_rewrite_to_sink
-rewritten_ast = full_rewrite_to_sink(kernel_ast)
-print(rewritten_ast)
-"""
-UOp(Ops.SINK, dtypes.void, arg=None, src=(
-  UOp(Ops.STORE, dtypes.void, arg=None, src=(
-    UOp(Ops.INDEX, dtypes.int.ptr(4), arg=None, src=(
-      UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(4), arg=0, src=()),
-      x3:=UOp(Ops.SPECIAL, dtypes.int, arg=('gidx0', 4), src=()),)),
-    UOp(Ops.ADD, dtypes.int, arg=None, src=(
-      UOp(Ops.LOAD, dtypes.int, arg=None, src=(
-        UOp(Ops.INDEX, dtypes.int.ptr(4), arg=None, src=(
-          UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(4), arg=1, src=()),
-           x3,)),)),
-      UOp(Ops.CONST, dtypes.int, arg=7, src=()),)),)),))
-"""
-# you can see at this point we are adding 7, not 3 and 4
-
-# with DEBUG=4, we can see the code.
-# since optimizations are on, it UPCASTed the operation, explicitly writing out all 4 +7s
-t_plus_3_plus_4.realize()
-"""
-void E_4n2(int* restrict data0, int* restrict data1) {
-  int val0 = *(data1+0);
-  int val1 = *(data1+1);
-  int val2 = *(data1+2);
-  int val3 = *(data1+3);
-  *(data0+0) = (val0+7);
-  *(data0+1) = (val1+7);
-  *(data0+2) = (val2+7);
-  *(data0+3) = (val3+7);
-}
-"""
-# the function name E_4n2 is "E" for elementwise op (as opposed to "r" for reduce op)
-# "4" for the size, and "n2" for name deduping (it's the 3rd function with the same E and 4 in this session)
-# when you print the name with DEBUG=2, you'll see the 4 is yellow, meaning that it's upcasted
-# if you run with NOOPT=1 ...
-"""
-void E_4n2(int* restrict data0, int* restrict data1) {
-  for (int ridx0 = 0; ridx0 < 4; ridx0++) {
-    int val0 = *(data1+ridx0);
-    *(data0+ridx0) = (val0+7);
-  }
-}
-"""
-# ... you get this unoptimized code with a loop and the 4 is blue (for global). the color code is in kernel.py
-
-# %% ********
-print("******* PART 3 *******")
-
-# now, we go even lower and understand UOps better and how the graph rewrite engine works.
-# it's much simpler than what's in LLVM or MLIR
-
-from tinygrad import dtypes
-from tinygrad.uop.ops import UOp, Ops
-
-# first, we'll construct some const UOps
-a = UOp(Ops.CONST, dtypes.int, arg=2)
-b = UOp(Ops.CONST, dtypes.int, arg=2)
-
-# if you have been paying attention, you should know these are the same Python object
-assert a is b
-
-# UOps support normal Python math operations, so a_plus_b expresses the spec for 2 + 2
-a_plus_b = a + b
-print(a_plus_b)
-"""
-UOp(Ops.ADD, dtypes.int, arg=None, src=(
-  x0:=UOp(Ops.CONST, dtypes.int, arg=2, src=()),
-   x0,))
-"""
-
-# we could actually render this 2+2 into a language like c and run it
-# or, we can use tinygrad's graph rewrite engine to "constant fold"
-
-from tinygrad.uop.ops import graph_rewrite, UPat, PatternMatcher
-
-# a `PatternMatcher` is a list of tuples. for each element in the list:
-# [0] is the pattern to match, and [1] is the function to run.
-# this function can return either a UOp to replace the pattern with, or None to not replace
-simple_pm = PatternMatcher([
-  (UPat(Ops.ADD, src=(UPat(Ops.CONST, name="c1"), UPat(Ops.CONST, name="c2"))),
-   lambda c1,c2: UOp(Ops.CONST, dtype=c1.dtype, arg=c1.arg+c2.arg)),
-])
-# this pattern matches the addition of two CONST and rewrites it into a single CONST UOp
-
-# to actually apply the pattern to a_plus_b, we use graph_rewrite
-a_plus_b_simplified = graph_rewrite(a_plus_b, simple_pm)
-print(a_plus_b_simplified)
-"""
-UOp(Ops.CONST, dtypes.int, arg=4, src=())
-"""
-# 2+2 is in fact, 4
-
-# we can also use syntactic sugar to write the pattern nicer
-simpler_pm = PatternMatcher([
-  (UPat.cvar("c1")+UPat.cvar("c2"), lambda c1,c2: c1.const_like(c1.arg+c2.arg))
-])
-assert graph_rewrite(a_plus_b, simple_pm) is graph_rewrite(a_plus_b, simpler_pm)
-# note again the use of is, UOps are immutable and globally unique
-
-# %% ********
-
-# that brings you to an understanding of the most core concepts in tinygrad
-# you can run this with VIZ=1 to use the web based graph rewrite explorer
-# hopefully now you understand it. the nodes in the graph are just UOps

docs/runtime.md

@@ -2,17 +2,17 @@
 
 tinygrad supports various runtimes, enabling your code to scale across a wide range of devices. The default runtime can be automatically selected based on the available hardware, or you can force a specific runtime to be default using environment variables (e.g., `CPU=1`).
 
-| Runtime | Description | Compiler Options | Requirements |
-|---------|-------------|------------------|--------------|
-| [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | nvrtc (default)<br>PTX (`NV_PTX=1`) | Ampere/Ada/Blackwell series GPUs.<br>You can select an interface via `NV_IFACE=(NVK\|PCI)`. See [NV interfaces](#nv-interfaces) for details. |
-| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`AMD_LLVM=1`)<br>HIP/COMGR (`AMD_HIP=1`) | RDNA2 or newer GPUs.<br>You can select an interface via `AMD_IFACE=(KFD\|PCI\|USB)`. See [AMD interfaces](#amd-interfaces) for details. |
-| [QCOM](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_qcom.py) | Provides acceleration for QCOM GPUs | - | 6xx series GPUs |
-| [METAL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_metal.py) | Utilizes Metal for acceleration on Apple devices | - | M1+ Macs; Metal 3.0+ for `bfloat` support |
-| [CUDA](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cuda.py) | Utilizes CUDA for acceleration on NVIDIA GPUs | nvrtc (default)<br> PTX (`CUDA_PTX=1`) | NVIDIA GPU with CUDA support |
-| [CL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cl.py) | Accelerates computations using OpenCL on GPUs | - | OpenCL 2.0 compatible device |
-| [CPU](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cpu.py) | Runs on CPU using the clang or llvm compiler | Clang JIT (default)<br>LLVM IR (`CPU_LLVM=1`) | `clang` compiler in system `PATH` |
-| [WEBGPU](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_webgpu.py) | Runs on GPU using the Dawn WebGPU engine (used in Google Chrome) | - | Dawn library installed and discoverable. Binaries: [pydawn v0.3.0](https://github.com/wpmed92/pydawn/releases/tag/v0.3.0) |
-
+| Runtime | Description | Requirements |
+|---------|-------------|--------------|
+| [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | Ampere/Ada series GPUs |
+| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | RDNA2/RDNA3/RDNA4 series GPUs |
+| [QCOM](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_qcom.py) | Provides acceleration for QCOM GPUs | 6xx series GPUs |
+| [METAL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_metal.py) | Utilizes Metal for acceleration on Apple devices | M1+ Macs; Metal 3.0+ for `bfloat` support |
+| [CUDA](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cuda.py) | Utilizes CUDA for acceleration on NVIDIA GPUs | NVIDIA GPU with CUDA support |
+| [GPU (OpenCL)](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_gpu.py) | Accelerates computations using OpenCL on GPUs | OpenCL 2.0 compatible device |
+| [CPU (C Code)](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cpu.py) | Runs on CPU using the clang compiler | `clang` compiler in system `PATH` |
+| [LLVM (LLVM IR)](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_llvm.py) | Runs on CPU using the LLVM compiler infrastructure | llvm libraries installed and findable |
+| [WEBGPU](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_webgpu.py) | Runs on GPU using the Dawn WebGPU engine (used in Google Chrome) | Dawn library installed and findable. Download binaries [here](https://github.com/wpmed92/pydawn/releases/tag/v0.1.6). |
 
 ## Interoperability
 
@@ -64,18 +64,3 @@ rawbuf_ptr = to_mv(cl_buf_desc_ptr, 0x100).cast('Q')[20] # offset 0xA0 is a raw
 # create tiny tensor
 tiny = Tensor.from_blob(rawbuf_ptr, (h*w*4,), dtype=dtypes.imagef((h,w)), device='QCOM')
 ```
-
-## AMD Interfaces
-AMD backend supports several interfaces for communicating with devices:
-
-* `KFD`: uses the amdgpu driver
-* `PCI`: uses the [AM driver](developer/am.md)
-* `USB`: USB3 interafce for asm24xx chips.
-
-You can force an interface by setting `AMD_IFACE` to one of these values. In the case of `AMD_IFACE=PCI`, this may unbind your GPU from the amdgpu driver.
-
-## NV Interfaces
-NV backend supports several interfaces for communicating with devices:
-
-* `NVK`: uses the nvidia driver
-* `PCI`: uses the [NV driver](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/support/nv/nvdev.py)

docs/tensor/creation.md

@@ -22,9 +22,7 @@
 ::: tinygrad.Tensor.rand
 ::: tinygrad.Tensor.rand_like
 ::: tinygrad.Tensor.randn
-::: tinygrad.Tensor.randn_like
 ::: tinygrad.Tensor.randint
-::: tinygrad.Tensor.randperm
 ::: tinygrad.Tensor.normal
 ::: tinygrad.Tensor.uniform
 ::: tinygrad.Tensor.scaled_uniform

docs/tensor/elementwise.md

@@ -35,7 +35,6 @@ Elementwise ops operate on a per element basis. They don't change the shape of t
 
 ::: tinygrad.Tensor.relu
 ::: tinygrad.Tensor.sigmoid
-::: tinygrad.Tensor.logsigmoid
 ::: tinygrad.Tensor.hardsigmoid
 ::: tinygrad.Tensor.elu
 ::: tinygrad.Tensor.celu
@@ -78,7 +77,6 @@ Elementwise ops operate on a per element basis. They don't change the shape of t
 ::: tinygrad.Tensor.minimum
 ::: tinygrad.Tensor.where
 ::: tinygrad.Tensor.copysign
-::: tinygrad.Tensor.logaddexp
 
 ## Casting Ops
 

docs/tensor/movement.md

@@ -18,7 +18,6 @@
 ::: tinygrad.Tensor.repeat_interleave
 ::: tinygrad.Tensor.split
 ::: tinygrad.Tensor.chunk
-::: tinygrad.Tensor.unfold
 ::: tinygrad.Tensor.meshgrid
 ::: tinygrad.Tensor.squeeze
 ::: tinygrad.Tensor.unsqueeze
@@ -26,6 +25,5 @@
 ::: tinygrad.Tensor.transpose
 ::: tinygrad.Tensor.flatten
 ::: tinygrad.Tensor.unflatten
-::: tinygrad.Tensor.diag
 ::: tinygrad.Tensor.roll
 ::: tinygrad.Tensor.rearrange

docs/tensor/ops.md

@@ -37,10 +37,8 @@
 ::: tinygrad.Tensor.scatter
 ::: tinygrad.Tensor.scatter_reduce
 ::: tinygrad.Tensor.masked_select
-::: tinygrad.Tensor.masked_fill
 ::: tinygrad.Tensor.sort
 ::: tinygrad.Tensor.topk
-::: tinygrad.Tensor.multinomial
 
 ## Neural Network (functional)
 

docs/tinybox.md

@@ -6,7 +6,7 @@ If you don't have a tinybox and you want one, see [tinygrad.org](https://tinygra
 
 ## Welcome
 
-Welcome to your tinybox! The tinybox is the universal system purpose-built for all AI infrastructure and workloads, from training to inference. The red box includes six 7900XTX GPUs, the green box includes six 4090 GPUs, and the green v2 box includes four 5090 GPUs. Whether you bought a red one or a green one, we want you to love it.
+Welcome to your tinybox! The tinybox is the universal system purpose-built for all AI infrastructure and workloads, from training to inference. The red box includes six 7900XTX GPUs, and the green box includes six 4090 GPUs. Whether you bought a red one or a green one, we want you to love it.
 
 We don't have a stupid cloud service, you don't have to create a tiny account to set it up, and we aren't tracking how you use the box. We're just happy you bought one. This petaflop is your petaflop.
 
@@ -47,8 +47,8 @@ Reboot after making these changes or restart the `displayservice.service` servic
 
 The [default tinybox image](https://github.com/tinygrad/tinyos) ships with tinygrad and PyTorch. While we develop tinygrad, the box is universal hardware. Use whatever framework you desire, run notebooks, download demos, install more things, train, inference, live, laugh, love, you aren't paying per hour for this box so the only limit is your imagination.
 
-## Building the OS image
+## tinychat
 
-The OS image is built using `ubuntu-image` from <https://github.com/tinygrad/tinyos>.
+Since LLMs are so popular, we ship with a built in tinygrad based chatbot using a LLaMA-3 finetune. Visit the IP (not the BMC IP) of your tinybox in a web browser on your computer or phone, and you'll find a friendly looking chat interface. This chatbot also provides an OpenAI compatible LLM API on that port, so you can script it.
 
-After cloning, run `make green` or `make red` to build a tinybox green or tinybox red image respectively.
+The conversations you have with this chatbot are between you and your tinybox. Also, the history in the web app is saved on the client, not the tinybox.

examples/beautiful_cartpole.py

@@ -78,7 +78,10 @@ if __name__ == "__main__":
 
   @TinyJit
   def get_action(obs:Tensor) -> Tensor:
+    # TODO: with no_grad
+    Tensor.no_grad = True
     ret = model(obs)[0].exp().multinomial().realize()
+    Tensor.no_grad = False
     return ret
 
   st, steps = time.perf_counter(), 0

examples/beautiful_cifar.py

@@ -2,19 +2,15 @@ import time
 start_tm = time.perf_counter()
 import math
 from typing import Tuple, cast
-from tinygrad import Tensor, nn, GlobalCounters, TinyJit, dtypes, Device
+import numpy as np
+from tinygrad import Tensor, nn, GlobalCounters, TinyJit, dtypes
 from tinygrad.helpers import partition, trange, getenv, Context
 from extra.lr_scheduler import OneCycleLR
 
-GPUS = [f'{Device.DEFAULT}:{i}' for i in range(getenv("GPUS", 1))]
-
-# override tinygrad defaults
 dtypes.default_float = dtypes.half
-Context(FUSE_ARANGE=1, FUSE_OPTIM=1).__enter__()
 
 # from https://github.com/tysam-code/hlb-CIFAR10/blob/main/main.py
 batchsize = getenv("BS", 1024)
-assert batchsize % len(GPUS) == 0, f"{batchsize=} is not a multiple of {len(GPUS)=}"
 bias_scaler = 64
 hyp = {
   'opt': {
@@ -71,7 +67,7 @@ class ConvGroup:
     cast(Tensor, self.norm2.weight).requires_grad = False
   def __call__(self, x:Tensor) -> Tensor:
     x =    self.norm1(self.conv1(x).max_pool2d().float()).cast(dtypes.default_float).quick_gelu()
-    return self.norm2(self.conv2(x).float()).cast(dtypes.default_float).quick_gelu() + x
+    return self.norm2(self.conv2(x).float()).cast(dtypes.default_float).quick_gelu()
 
 class SpeedyConvNet:
   def __init__(self):
@@ -82,25 +78,23 @@ class SpeedyConvNet:
     self.linear = nn.Linear(depths['block3'], depths['num_classes'], bias=False)
   def __call__(self, x:Tensor) -> Tensor:
     x = self.whiten(x).quick_gelu()
-    # ************* HACKS *************
-    x = x.pad((1,0,0,1)) # TODO: this pad should not be here! copied from hlb_cifar10 for speed
-    # ************* HACKS *************
     x = x.sequential([self.conv_group_1, self.conv_group_2, self.conv_group_3])
     return self.linear(x.max(axis=(2,3))) * hyp['opt']['scaling_factor']
 
 if __name__ == "__main__":
   # *** dataset ***
   X_train, Y_train, X_test, Y_test = nn.datasets.cifar()
+  # TODO: without this line indexing doesn't fuse!
+  X_train, Y_train, X_test, Y_test = [x.contiguous() for x in [X_train, Y_train, X_test, Y_test]]
   cifar10_std, cifar10_mean = X_train.float().std_mean(axis=(0, 2, 3))
-  def preprocess(X:Tensor) -> Tensor: return ((X - cifar10_mean.view(1, -1, 1, 1)) / cifar10_std.view(1, -1, 1, 1)).cast(dtypes.default_float)
+  def preprocess(X:Tensor, Y:Tensor) -> Tuple[Tensor, Tensor]:
+    return ((X - cifar10_mean.view(1, -1, 1, 1)) / cifar10_std.view(1, -1, 1, 1)).cast(dtypes.default_float), Y.one_hot(depths['num_classes'])
 
   # *** model ***
   model = SpeedyConvNet()
   state_dict = nn.state.get_state_dict(model)
-  if len(GPUS) > 1:
-    cifar10_std.to_(GPUS)
-    cifar10_mean.to_(GPUS)
-    for x in state_dict.values(): x.to_(GPUS)
+
+  #for k,v in nn.state.torch_load("/tmp/cifar_net.pt").items(): print(k)
 
   params_bias, params_non_bias = partition(state_dict.items(), lambda x: 'bias' in x[0])
   opt_bias     = nn.optim.SGD([x[1] for x in params_bias],     lr=0.01, momentum=.85, nesterov=True, weight_decay=hyp['opt']['bias_decay'])
@@ -117,44 +111,48 @@ if __name__ == "__main__":
   lr_sched_bias     = OneCycleLR(opt_bias,     max_lr=hyp['opt']['bias_lr'],     pct_start=pct_start, div_factor=initial_div_factor, final_div_factor=1./(initial_div_factor*final_lr_ratio), total_steps=total_train_steps)
   lr_sched_non_bias = OneCycleLR(opt_non_bias, max_lr=hyp['opt']['non_bias_lr'], pct_start=pct_start, div_factor=initial_div_factor, final_div_factor=1./(initial_div_factor*final_lr_ratio), total_steps=total_train_steps)
 
-  def loss_fn(out:Tensor, Y:Tensor) -> Tensor:
-    ret = out.sparse_categorical_crossentropy(Y, reduction='none', label_smoothing=0.2)
-    return ret.mul(hyp['opt']['loss_scale_scaler']*loss_batchsize_scaler).sum().div(hyp['opt']['loss_scale_scaler'])
+  def loss_fn(out, Y):
+    return out.cross_entropy(Y, reduction='none', label_smoothing=0.2).mul(hyp['opt']['loss_scale_scaler']*loss_batchsize_scaler).sum().div(hyp['opt']['loss_scale_scaler'])
 
   @TinyJit
   @Tensor.train()
   def train_step(idxs:Tensor) -> Tensor:
-    X, Y = X_train[idxs], Y_train[idxs]
-    if len(GPUS) > 1:
-      X.shard_(GPUS, axis=0)
-      Y.shard_(GPUS, axis=0)
-    out = model(preprocess(X))
+    with Context(SPLIT_REDUCEOP=0, FUSE_ARANGE=1):
+      X = X_train[idxs]
+      Y = Y_train[idxs].realize(X)
+    X, Y = preprocess(X, Y)
+    out = model(X)
     loss = loss_fn(out, Y)
     opt.zero_grad()
     loss.backward()
-    return (loss / (batchsize*loss_batchsize_scaler)).realize(*opt.schedule_step(),
-                                                              *lr_sched_bias.schedule_step(), *lr_sched_non_bias.schedule_step())
+    opt.step()
+    lr_sched_bias.step()
+    lr_sched_non_bias.step()
+    return loss / (batchsize*loss_batchsize_scaler)
 
   eval_batchsize = 2500
   @TinyJit
+  @Tensor.test()
   def val_step() -> Tuple[Tensor, Tensor]:
+    # TODO with Tensor.no_grad()
+    Tensor.no_grad = True
     loss, acc = [], []
     for i in range(0, X_test.size(0), eval_batchsize):
-      X, Y = X_test[i:i+eval_batchsize], Y_test[i:i+eval_batchsize]
-      if len(GPUS) > 1:
-        X.shard_(GPUS, axis=0)
-        Y.shard_(GPUS, axis=0)
-      out = model(preprocess(X))
+      X, Y = preprocess(X_test[i:i+eval_batchsize], Y_test[i:i+eval_batchsize])
+      out = model(X)
       loss.append(loss_fn(out, Y))
-      acc.append((out.argmax(-1) == Y).sum() / eval_batchsize)
-    return Tensor.stack(*loss).mean() / (batchsize*loss_batchsize_scaler), Tensor.stack(*acc).mean()
-
-  Tensor.manual_seed(1337)
-  num_train_samples = X_train.shape[0]
+      acc.append((out.argmax(-1).one_hot(depths['num_classes']) * Y).sum() / eval_batchsize)
+    ret = Tensor.stack(*loss).mean() / (batchsize*loss_batchsize_scaler), Tensor.stack(*acc).mean()
+    Tensor.no_grad = False
+    return ret
 
+  np.random.seed(1337)
   for epoch in range(math.ceil(hyp['misc']['train_epochs'])):
+    # TODO: move to tinygrad
     gst = time.perf_counter()
-    tidxs = Tensor.randperm(num_train_samples, dtype='int')[:num_steps_per_epoch*batchsize].reshape(num_steps_per_epoch, batchsize)
+    idxs = np.arange(X_train.shape[0])
+    np.random.shuffle(idxs)
+    tidxs = Tensor(idxs, dtype='int')[:num_steps_per_epoch*batchsize].reshape(num_steps_per_epoch, batchsize)  # NOTE: long doesn't fold
     train_loss:float = 0
     for epoch_step in (t:=trange(num_steps_per_epoch)):
       st = time.perf_counter()

examples/beautiful_mnist.py

@@ -1,12 +1,12 @@
-# model based off https://medium.com/data-science/going-beyond-99-mnist-handwritten-digits-recognition-cfff96337392
-from typing import Callable
+# model based off https://towardsdatascience.com/going-beyond-99-mnist-handwritten-digits-recognition-cfff96337392
+from typing import List, Callable
 from tinygrad import Tensor, TinyJit, nn, GlobalCounters
 from tinygrad.helpers import getenv, colored, trange
 from tinygrad.nn.datasets import mnist
 
 class Model:
   def __init__(self):
-    self.layers: list[Callable[[Tensor], Tensor]] = [
+    self.layers: List[Callable[[Tensor], Tensor]] = [
       nn.Conv2d(1, 32, 5), Tensor.relu,
       nn.Conv2d(32, 32, 5), Tensor.relu,
       nn.BatchNorm(32), Tensor.max_pool2d,
@@ -21,17 +21,20 @@ if __name__ == "__main__":
   X_train, Y_train, X_test, Y_test = mnist(fashion=getenv("FASHION"))
 
   model = Model()
-  opt = (nn.optim.Adam if not getenv("MUON") else nn.optim.Muon)(nn.state.get_parameters(model))
+  opt = nn.optim.Adam(nn.state.get_parameters(model))
 
   @TinyJit
   @Tensor.train()
   def train_step() -> Tensor:
     opt.zero_grad()
     samples = Tensor.randint(getenv("BS", 512), high=X_train.shape[0])
+    # TODO: this "gather" of samples is very slow. will be under 5s when this is fixed
     loss = model(X_train[samples]).sparse_categorical_crossentropy(Y_train[samples]).backward()
-    return loss.realize(*opt.schedule_step())
+    opt.step()
+    return loss
 
   @TinyJit
+  @Tensor.test()
   def get_test_acc() -> Tensor: return (model(X_test).argmax(axis=1) == Y_test).mean()*100
 
   test_acc = float('nan')

examples/benchmark_onnx.py

@@ -1,10 +1,11 @@
-import sys, time
+import sys, onnx, time, pickle
 from tinygrad import TinyJit, GlobalCounters, fetch, getenv
 from tinygrad.frontend.onnx import OnnxRunner
 from extra.onnx_helpers import get_example_inputs, validate
 
 def load_onnx_model(onnx_file):
-  run_onnx = OnnxRunner(onnx_file)
+  onnx_model = onnx.load(onnx_file)
+  run_onnx = OnnxRunner(onnx_model)
   run_onnx_jit = TinyJit(lambda **kwargs: next(iter(run_onnx({k:v.to(None) for k,v in kwargs.items()}).values())), prune=True, optimize=True)
   return run_onnx_jit, run_onnx.graph_inputs
 

examples/coder.py

@@ -4,7 +4,7 @@ sys.path.append(os.getcwd())
 
 from io import StringIO
 from contextlib import redirect_stdout
-from tinygrad import Tensor, nn
+from tinygrad import Tensor, nn, Device, dtypes
 from tinygrad.helpers import Timing, colored, getenv, fetch
 from extra.models.llama import Transformer, convert_from_huggingface, fix_bf16
 from sentencepiece import SentencePieceProcessor
@@ -23,6 +23,8 @@ def create_fixed_tokenizer(output_file):
 # echo -en "write 2+2\nwrite hello world\ny\n" | TEMP=0 python3 examples/coder.py
 
 if __name__ == "__main__":
+  Tensor.no_grad = True
+
   # https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B/blob/main/config.json
   with Timing("create model: "):
     model = Transformer(4096, 14336, n_heads=32, n_layers=32, norm_eps=1e-5, vocab_size=32002, n_kv_heads=8, max_context=4096, jit=getenv("JIT", 1))
@@ -32,8 +34,8 @@ if __name__ == "__main__":
     part2 = nn.state.torch_load(fetch("https://huggingface.co/teknium/OpenHermes-2.5-Mistral-7B/resolve/main/pytorch_model-00002-of-00002.bin?download=true"))
 
   with Timing("weights -> model: "):
-    nn.state.load_state_dict(model, fix_bf16(convert_from_huggingface(part1, 32, 32, 8)), strict=False)
-    nn.state.load_state_dict(model, fix_bf16(convert_from_huggingface(part2, 32, 32, 8)), strict=False)
+    nn.state.load_state_dict(model, fix_bf16(convert_from_huggingface(part1, model, 32, 8)), strict=False)
+    nn.state.load_state_dict(model, fix_bf16(convert_from_huggingface(part2, model, 32, 8)), strict=False)
 
   if not os.path.isfile("/tmp/tokenizer.model"): create_fixed_tokenizer("/tmp/tokenizer.model")
   spp = SentencePieceProcessor(model_file="/tmp/tokenizer.model")

examples/compile_tensorflow.py

@@ -10,7 +10,6 @@ import tensorflow as tf
 import tf2onnx
 from tinygrad.frontend.onnx import OnnxRunner
 from tinygrad.tensor import Tensor
-from tinygrad.helpers import to_mv
 from extra.export_model import export_model_clang, compile_net, jit_model
 
 def get_uncompiled_model2(dataset_size=32, output_size=4):
@@ -26,7 +25,7 @@ class TinyOnnx:
   def __init__(self, keras_model):
     input_signature = [tf.TensorSpec([1,32], tf.float32, name='x')]
     onnx_model, _ = tf2onnx.convert.from_keras(keras_model, input_signature, opset=13)
-    self.run_onnx = OnnxRunner(Tensor(onnx_model.SerializeToString(), device="PYTHON"))
+    self.run_onnx = OnnxRunner(onnx_model)
 
   def forward(self, x):
     return self.run_onnx({"x": x}, debug=False)['predictions']
@@ -48,8 +47,8 @@ def compile_onnx_model(onnx_model):
   cprog.append("void initialize(float *weights) {")
   weights = bytes()
   for name,cl in bufs_to_save.items():
-    cprog.append(f"memcpy({name}, weights + {len(weights)//4}, {cl._buf.size});")
-    weights += bytes(to_mv(cl._buf.va_addr, cl._buf.size))
+    cprog.append(f"memcpy({name}, weights + {len(weights)//4}, {len(cl._buf)*4});")
+    weights += bytes(cl._buf)
   cprog.append("}")
 
   # write the weights to disk

examples/conversation.py

@@ -159,6 +159,7 @@ def init_vits(
   text_mapper = TextMapper(apply_cleaners=True, symbols=symbols)
 
   # Load the model.
+  Tensor.no_grad = True
   if seed is not None:
     Tensor.manual_seed(seed)
     np.random.seed(seed)
@@ -220,6 +221,7 @@ def mp_output_stream(q: mp.Queue, counter: mp.Value, num_channels: int, sample_r
 if __name__ == "__main__":
   import nltk
   nltk.download("punkt")
+  Tensor.no_grad = True
   # Parse CLI arguments
   parser = argparse.ArgumentParser("Have a tiny conversation with tinygrad")
 

examples/gpt2.py

@@ -3,11 +3,10 @@ import os, argparse, contextlib
 from typing import Optional, Union
 with contextlib.suppress(ImportError): import tiktoken
 from tinygrad import Tensor, TinyJit, Device, GlobalCounters, Variable, dtypes
-from tinygrad.uop.ops import UOp
+from tinygrad.ops import UOp
 from tinygrad.helpers import Timing, DEBUG, JIT, getenv, fetch, colored, trange
 from tinygrad.nn import Embedding, Linear, LayerNorm
 from tinygrad.nn.state import gguf_load, torch_load, load_state_dict, get_state_dict
-from extra.bench_log import BenchEvent, WallTimeEvent
 
 MAX_CONTEXT = getenv("MAX_CONTEXT", 128)
 HALF = getenv("HALF")
@@ -26,8 +25,8 @@ class Attention:
       start_pos = start_pos.val
 
     if HALF: x = x.half()
-    xqkv = self.c_attn(x).reshape(None, None, 3, self.n_heads, self.head_dim)
-    xq, xk, xv = [xqkv[:, :, i, :, :] for i in range(3)]
+    xqkv = self.c_attn(x)
+    xq, xk, xv = [xqkv.shrink((None, None, (i*self.dim, (i+1)*self.dim))).reshape(None, None, self.n_heads, self.head_dim) for i in range(3)]
     bsz, seqlen, _, _ = xq.shape
 
     # create kv cache
@@ -35,11 +34,11 @@ class Attention:
       self.cache_kv = Tensor.zeros(2, bsz, MAX_CONTEXT, self.n_heads, self.head_dim, dtype=x.dtype).contiguous().realize()
 
     # update the cache
-    self.cache_kv[:, :, start_pos:start_pos+seqlen, :, :].assign(Tensor.stack(xk, xv)).realize()
+    self.cache_kv.shrink((None, None,(start_pos,start_pos+seqlen),None,None)).assign(Tensor.stack(xk, xv)).realize()
 
     if start_pos > 0:
-      keys = self.cache_kv[0][:, :start_pos+seqlen, :, :]
-      values = self.cache_kv[1][:, :start_pos+seqlen, :, :]
+      keys = self.cache_kv[0].shrink((None, (0, start_pos+seqlen), None, None))
+      values = self.cache_kv[1].shrink((None, (0, start_pos+seqlen), None, None))
     else:
       keys = xk
       values = xv
@@ -64,7 +63,7 @@ class TransformerBlock:
 
   def __call__(self, x:Tensor, start_pos:Variable, mask:Optional[Tensor]):
     h = x + self.attn(self.ln_1(x), start_pos, mask).float()
-    return (h + self.mlp(self.ln_2(h))).contiguous()
+    return (h + self.mlp(self.ln_2(h)))
 
 class Transformer:
   def __init__(self, dim, n_heads, n_layers, norm_eps, vocab_size, max_seq_len=1024):
@@ -85,10 +84,7 @@ class Transformer:
       seqlen = tokens.shape[1]
       tok_emb = self.wte(tokens)
 
-    # not symbolic when consuming the prompt
-    selected_pos = (0, seqlen) if start_pos.val == 0 else (start_pos, start_pos+1)
-    pos_emb = self.wpe(self.allpos.shrink((None, selected_pos)))
-
+    pos_emb = self.wpe(self.allpos.shrink((None, (start_pos, start_pos+seqlen))))
     h = tok_emb + pos_emb
 
     if HALF: h = h.half()
@@ -138,12 +134,11 @@ class GPT2:
     # lm head and wte are tied
     weights['lm_head.weight'] = weights['wte.weight']
 
-    with WallTimeEvent(BenchEvent.LOAD_WEIGHTS):
-      load_state_dict(model, weights)
+    load_state_dict(model, weights)
 
-      if HALF:
-        for l in get_state_dict(model).values():
-          l.replace(l.half().realize())
+    if HALF:
+      for l in get_state_dict(model).values():
+        l.replace(l.half().realize())
 
     return GPT2(model, tokenizer)
 
@@ -172,8 +167,7 @@ class GPT2:
       return key
     state_dict = { _remap_gguf_key(k): v for k, v in state_dict.items() }
     model = Transformer(**gpt2_params)
-    with WallTimeEvent(BenchEvent.LOAD_WEIGHTS):
-      load_state_dict(model, state_dict)
+    load_state_dict(model, state_dict)
     return GPT2(model, tiktoken.get_encoding("gpt2"))
 
   def __init__(self, model, tokenizer):
@@ -181,7 +175,6 @@ class GPT2:
     self.tokenizer = tokenizer
 
   def generate(self, prompt:str, max_length:int, temperature:float, timing:bool=False, batch_size:int=1):
-    step_times = []
     prompt_tokens = self.tokenizer.encode(prompt, allowed_special={"<|endoftext|>"})
     toks = [prompt_tokens[:] for _ in range(batch_size)]
     start_pos = 0
@@ -189,27 +182,22 @@ class GPT2:
       GlobalCounters.reset()
       if timing: print("")
       st = GlobalCounters.time_sum_s
-      with Timing("ran model in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on {Device.DEFAULT}" if DEBUG>=2 else "")+
+      with Timing("ran model in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on GPU" if DEBUG>=2 else "")+
                   f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
                   (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None, enabled=timing):
-        with WallTimeEvent(BenchEvent.STEP):
-          if batch_size == 1 and len(toks[0][start_pos:]) == 1:
-            tokens = Variable("tokens", 0, VOCAB_SIZE-1).bind(toks[0][start_pos])
-          else:
-            tokens = Tensor([x[start_pos:] for x in toks])
-          tok = self.model(tokens, Variable("start_pos", 1 if start_pos else 0, MAX_CONTEXT-1).bind(start_pos), temperature).tolist()
-      step_times.append((GlobalCounters.time_sum_s-st)*1e3)
+        if batch_size == 1 and len(toks[0][start_pos:]) == 1:
+          tokens = Variable("tokens", 0, VOCAB_SIZE).bind(toks[0][start_pos])
+        else:
+          tokens = Tensor([x[start_pos:] for x in toks])
+        tok = self.model(tokens, Variable("start_pos", 1 if start_pos else 0, MAX_CONTEXT-1).bind(start_pos), temperature).tolist()
       start_pos = len(toks[0])
       for i,t in enumerate(tok): toks[i].append(t)
-
-    if (assert_time:=getenv("ASSERT_MIN_STEP_TIME")):
-      min_time = min(step_times)
-      assert min_time < assert_time, f"Speed regression, expected min step time of < {assert_time} ms but took: {min_time} ms"
     return [self.tokenizer.decode(x) for x in toks]
 
 # **** main code ****
 
 if __name__ == "__main__":
+  Tensor.no_grad = True
   print(f"using {Device.DEFAULT} backend")
   default_prompt = "What is the answer to life, the universe, and everything?"
 

examples/handcode_opt.py

@@ -0,0 +1,133 @@
+from extra.models.resnet import ResNet50
+from extra.mcts_search import mcts_search
+from examples.mlperf.helpers import get_mlperf_bert_model
+from tinygrad import Tensor, Device, dtypes, nn
+from tinygrad.codegen.kernel import Kernel
+from tinygrad.codegen.heuristic import hand_coded_optimizations
+from tinygrad.ops import Ops, sym_infer
+from tinygrad.device import Compiled
+from tinygrad.engine.search import beam_search, bufs_from_lin
+from tinygrad.helpers import DEBUG, ansilen, getenv, colored, TRACEMETA
+from extra.optimization.helpers import time_linearizer
+
+def get_sched_resnet():
+  mdl = ResNet50()
+  optim = (nn.optim.LARS if getenv("LARS") else nn.optim.SGD)(nn.state.get_parameters(mdl))
+  BS = getenv("BS", 64)
+
+  # run model twice to get only what changes, these are the kernels of the model
+  for _ in range(2):
+    out = mdl(Tensor.empty(BS, 3, 224, 224))
+    targets = [out]
+    if getenv("BACKWARD"):
+      optim.zero_grad()
+      out.sparse_categorical_crossentropy(Tensor.empty(BS, dtype=dtypes.int)).backward()
+      targets += [x for x in optim.schedule_step()]
+    sched = Tensor.schedule(*targets)
+    print(f"schedule length {len(sched)}")
+  return sched
+
+def get_sched_bert():
+  mdl = get_mlperf_bert_model()
+  optim = nn.optim.LAMB(nn.state.get_parameters(mdl))
+
+  # fake data
+  BS = getenv("BS", 9)
+  input_ids = Tensor.empty((BS, 512), dtype=dtypes.float32)
+  segment_ids = Tensor.empty((BS, 512), dtype=dtypes.float32)
+  attention_mask = Tensor.empty((BS, 512), dtype=dtypes.default_float)
+  masked_positions = Tensor.empty((BS, 76), dtype=dtypes.float32)
+  masked_lm_ids = Tensor.empty((BS, 76), dtype=dtypes.float32)
+  masked_lm_weights = Tensor.empty((BS, 76), dtype=dtypes.float32)
+  next_sentence_labels = Tensor.empty((BS, 1), dtype=dtypes.float32)
+
+  # run model twice to get only what changes, these are the kernels of the model
+  for _ in range(2):
+    lm_logits, seq_relationship_logits = mdl(input_ids, attention_mask, masked_positions, segment_ids)
+    targets = [lm_logits, seq_relationship_logits]
+    if getenv("BACKWARD"):
+      optim.zero_grad()
+      loss = mdl.loss(lm_logits, seq_relationship_logits, masked_lm_ids, masked_lm_weights, next_sentence_labels)
+      # ignore grad norm and loss scaler for now
+      loss.backward()
+      targets += [x for x in optim.schedule_step()]
+    sched = Tensor.schedule(*targets)
+    print(f"schedule length {len(sched)}")
+  return sched
+
+if __name__ == "__main__":
+  if getenv("HALF", 1):
+    dtypes.default_float = dtypes.half
+
+  # the device we are optimizing for
+  device: Compiled = Device[Device.DEFAULT]
+  if getenv("BACKWARD"): Tensor.training = True
+  print(f"optimizing for {Device.DEFAULT}")
+
+  sched = globals()[f"get_sched_{getenv('MODEL', 'resnet')}"]()
+  sched = [x for x in sched if x.ast.op is Ops.SINK]
+
+  # focus on one kernel
+  if getenv("KERNEL", -1) >= 0: sched = sched[getenv("KERNEL", -1):getenv("KERNEL", -1)+1]
+
+  # work with the schedule
+  total_tm = 0
+  running_gflops = 0
+  usage = {}
+  for i,si in enumerate(sched):
+    if DEBUG >= 3: print(si.ast)
+
+    rawbufs = bufs_from_lin(Kernel(si.ast))
+
+    # "linearize" the op into uops in different ways
+    lins: list[tuple[Kernel, str]] = []
+
+    # always try hand coded opt
+    lin = Kernel(si.ast, opts=device.renderer)
+    lin.apply_opts(hand_coded_optimizations(lin))
+    lins.append((lin, "HC"))
+
+    # maybe try tensor cores
+    lin = Kernel(si.ast, opts=device.renderer)
+    if lin.apply_tensor_cores():
+      lins.append((lin, "TC"))
+
+    # try a beam search
+    if beam:=getenv("BEAM"):
+      lin = Kernel(si.ast, opts=device.renderer)
+      lin = beam_search(lin, rawbufs, beam, bool(getenv("BEAM_ESTIMATE", 1)))
+      lins.append((lin, "BEAM"))
+
+    # try MCTS
+    if mcts:=getenv("MCTS"):
+      lin = Kernel(si.ast, opts=device.renderer)
+      lin = mcts_search(lin, rawbufs, mcts)
+      lins.append((lin, "MCTS"))
+
+    # benchmark the programs
+    choices = []
+    for lin, nm in lins:
+      tm = time_linearizer(lin, rawbufs, allow_test_size=False, cnt=10, disable_cache=True)
+      ops = (prg:=lin.to_program()).estimates.ops
+      gflops = sym_infer(ops, {k:k.min for k in lin.ast.variables()})*1e-9/tm
+      choices.append((tm, gflops, lin, prg, nm))
+
+    sorted_choices = sorted(choices, key=lambda x: x[0])
+    if DEBUG >= 1: # print all kernels
+      for tm, gflops, lin, prg, nm in choices:
+        print(f"                 kernel {i:2d} {lin.name+' '*(37-ansilen(lin.name))} {str(prg.global_size):18s} {str(prg.local_size):12s} takes {tm*1000:7.2f} ms, {gflops:6.0f} GFLOPS -- {colored(nm, 'green') if lin is sorted_choices[0][2] else nm}")
+
+    tm, gflops, lin, prg, nm = sorted_choices[0]
+    if getenv("SRC"):
+      print(si.ast)
+      print(lin.applied_opts)
+      print(lin.to_program().src)
+    total_tm += tm
+    running_gflops += gflops * tm
+    if (key := str([str(m) for m in si.metadata])) not in usage: usage[key] = (0, 0)
+    usage[key] = (usage[key][0] + tm, usage[key][1] + 1)
+    print(f"*** {total_tm*1000:7.2f} ms : kernel {i:2d} {lin.name+' '*(37-ansilen(lin.name))} {str(prg.global_size):18s} {str(prg.local_size):12s} takes {tm*1000:7.2f} ms, {gflops:6.0f} GFLOPS {[repr(m) if TRACEMETA >= 2 else str(m) for m in si.metadata]}")
+  print(f"******* total {total_tm*1000:.2f} ms, {running_gflops/total_tm:6.0f} GFLOPS")
+  print("usage:")
+  for k in sorted(usage, key=lambda x: -usage[x][0])[:10]:
+    print(f"{usage[k][0]*1000:.2f} ms: {k} ({usage[k][1]} times)")

examples/hlb_cifar10.py

@@ -7,11 +7,10 @@ import random, time
 import numpy as np
 from typing import Optional
 from extra.lr_scheduler import OneCycleLR
-from tinygrad import nn, dtypes, Tensor, Device, GlobalCounters, TinyJit, Variable
-from tinygrad.nn.state import get_state_dict
+from tinygrad import nn, dtypes, Tensor, Device, GlobalCounters, TinyJit
+from tinygrad.nn.state import get_state_dict, get_parameters
 from tinygrad.nn import optim
 from tinygrad.helpers import Context, BEAM, WINO, getenv, colored, prod
-from extra.bench_log import BenchEvent, WallTimeEvent
 
 cifar_mean = [0.4913997551666284, 0.48215855929893703, 0.4465309133731618]
 cifar_std = [0.24703225141799082, 0.24348516474564, 0.26158783926049628]
@@ -118,7 +117,7 @@ class SpeedyResNet:
 # hyper-parameters were exactly the same as the original repo
 bias_scaler = 58
 hyp = {
-  'seed' : 201,
+  'seed' : 209,
   'opt': {
     'bias_lr':            1.76 * bias_scaler/512,
     'non_bias_lr':        1.76 / 512,
@@ -145,7 +144,6 @@ hyp = {
   },
 }
 
-@Context(FUSE_ARANGE=getenv("FUSE_ARANGE", 1))
 def train_cifar():
 
   def set_seed(seed):
@@ -202,38 +200,24 @@ def train_cifar():
     idx_y = Tensor.arange(H, dtype=dtypes.int32).reshape((1,1,H,1))
     return (idx_x >= low_x) * (idx_x < (low_x + mask_size)) * (idx_y >= low_y) * (idx_y < (low_y + mask_size))
 
-  # Similar, but different enough.
-  def make_random_crop_indices(shape, mask_size) -> Tensor:
-    BS, _, H, W = shape
-    low_x = Tensor.randint(BS, low=0, high=W-mask_size).reshape(BS,1,1,1)
-    low_y = Tensor.randint(BS, low=0, high=H-mask_size).reshape(BS,1,1,1)
-    idx_x = Tensor.arange(mask_size, dtype=dtypes.int32).reshape((1,1,1,mask_size))
-    idx_y = Tensor.arange(mask_size, dtype=dtypes.int32).reshape((1,1,mask_size,1))
-    return low_x, low_y, idx_x, idx_y
-
   def random_crop(X:Tensor, crop_size=32):
-    Xs, Ys, Xi, Yi = make_random_crop_indices(X.shape, crop_size)
-    return X.gather(-1, (Xs + Xi).expand(-1, 3, X.shape[2], -1)).gather(-2, ((Ys+Yi).expand(-1, 3, crop_size, crop_size)))
+    mask = make_square_mask(X.shape, crop_size)
+    mask = mask.expand((-1,3,-1,-1))
+    X_cropped = Tensor(X.numpy()[mask.numpy()])
+    return X_cropped.reshape((-1, 3, crop_size, crop_size))
 
-  def cutmix(X, Y, order, mask_size=3):
+  def cutmix(X:Tensor, Y:Tensor, mask_size=3):
+    # fill the square with randomly selected images from the same batch
     mask = make_square_mask(X.shape, mask_size)
-    X_patch, Y_patch = X[order], Y[order]
+    order = list(range(0, X.shape[0]))
+    random.shuffle(order)
+    X_patch = Tensor(X.numpy()[order], device=X.device, dtype=X.dtype)
+    Y_patch = Tensor(Y.numpy()[order], device=Y.device, dtype=Y.dtype)
     X_cutmix = mask.where(X_patch, X)
     mix_portion = float(mask_size**2)/(X.shape[-2]*X.shape[-1])
     Y_cutmix = mix_portion * Y_patch + (1. - mix_portion) * Y
     return X_cutmix, Y_cutmix
 
-  @TinyJit
-  def augmentations(X:Tensor, Y:Tensor):
-    perms = Tensor.randperm(X.shape[0], device=X.device) # We reuse perms for cutmix, because they are expensivne to generate
-    if getenv("RANDOM_CROP", 1):
-      X = random_crop(X, crop_size=32)
-    if getenv("RANDOM_FLIP", 1):
-      # NOTE: RANGEIFY=1 needs this contiguous or the X[perms] is very slow
-      X = (Tensor.rand(X.shape[0],1,1,1) < 0.5).where(X.flip(-1), X).contiguous() # flip LR
-    X, Y = X[perms], Y[perms]
-    return X, Y, *cutmix(X, Y, perms, mask_size=hyp['net']['cutmix_size'])
-
   # the operations that remain inside batch fetcher is the ones that involves random operations
   def fetch_batches(X_in:Tensor, Y_in:Tensor, BS:int, is_train:bool):
     step, epoch = 0, 0
@@ -241,16 +225,28 @@ def train_cifar():
       st = time.monotonic()
       X, Y = X_in, Y_in
       if is_train:
-        X, Y, X_cm, Y_cm = augmentations(X, Y)
-        if getenv("CUTMIX", 1) and step >= hyp['net']['cutmix_steps']: X, Y = X_cm, Y_cm
+        # TODO: these are not jitted
+        if getenv("RANDOM_CROP", 1):
+          X = random_crop(X, crop_size=32)
+        if getenv("RANDOM_FLIP", 1):
+          X = (Tensor.rand(X.shape[0],1,1,1) < 0.5).where(X.flip(-1), X) # flip LR
+        if getenv("CUTMIX", 1):
+          if step >= hyp['net']['cutmix_steps']:
+            X, Y = cutmix(X, Y, mask_size=hyp['net']['cutmix_size'])
+        order = list(range(0, X.shape[0]))
+        random.shuffle(order)
+        X, Y = X.numpy()[order], Y.numpy()[order]
+      else:
+        X, Y = X.numpy(), Y.numpy()
       et = time.monotonic()
       print(f"shuffling {'training' if is_train else 'test'} dataset in {(et-st)*1e3:.2f} ms ({epoch=})")
-
-      vi = Variable("i", 0, (full_batches := (X.shape[0] // BS) * BS) - BS)
-      for i in range(0, full_batches, BS):
+      for i in range(0, X.shape[0], BS):
+        # pad the last batch  # TODO: not correct for test
+        batch_end = min(i+BS, Y.shape[0])
+        x = Tensor(X[batch_end-BS:batch_end], device=X_in.device, dtype=X_in.dtype)
+        y = Tensor(Y[batch_end-BS:batch_end], device=Y_in.device, dtype=Y_in.dtype)
         step += 1
-        vib = vi.bind(i)
-        yield X[vib:vib+BS], Y[vib:vib+BS]
+        yield x, y
       epoch += 1
       if not is_train: break
 
@@ -270,10 +266,13 @@ def train_cifar():
 
     @TinyJit
     def update(self, net, decay):
+      # TODO with Tensor.no_grad()
+      Tensor.no_grad = True
       for net_ema_param, (param_name, net_param) in zip(get_state_dict(self.net_ema).values(), get_state_dict(net).items()):
         # batchnorm currently is not being tracked
         if not ("num_batches_tracked" in param_name) and not ("running" in param_name):
           net_ema_param.assign(net_ema_param.detach()*decay + net_param.detach()*(1.-decay)).realize()
+      Tensor.no_grad = False
 
   set_seed(getenv('SEED', hyp['seed']))
 
@@ -356,7 +355,7 @@ def train_cifar():
 
   # https://www.anandtech.com/show/16727/nvidia-announces-geforce-rtx-3080-ti-3070-ti-upgraded-cards-coming-in-june
   # 136 TFLOPS is the theoretical max w float16 on 3080 Ti
-  step_times = []
+
   model_ema: Optional[modelEMA] = None
   projected_ema_decay_val = hyp['ema']['decay_base'] ** hyp['ema']['every_n_steps']
   i = 0
@@ -396,35 +395,27 @@ def train_cifar():
       if STEPS == 0 or i == STEPS: break
 
       GlobalCounters.reset()
+      X, Y = next(batcher)
+      if len(GPUS) > 1:
+        X.shard_(GPUS, axis=0)
+        Y.shard_(GPUS, axis=0)
 
-      with WallTimeEvent(BenchEvent.STEP):
-        X, Y = next(batcher)
-        if len(GPUS) > 1:
-          X.shard_(GPUS, axis=0)
-          Y.shard_(GPUS, axis=0)
-
-        with Context(BEAM=getenv("LATEBEAM", BEAM.value), WINO=getenv("LATEWINO", WINO.value)):
-          loss = train_step_jitted(model, optim.OptimizerGroup(opt_bias, opt_non_bias), [lr_sched_bias, lr_sched_non_bias], X, Y)
-          et = time.monotonic()
-          loss_cpu = loss.numpy()
-        # EMA for network weights
-        if getenv("EMA") and i > hyp['ema']['steps'] and (i+1) % hyp['ema']['every_n_steps'] == 0:
-          if model_ema is None:
-            model_ema = modelEMA(W, model)
-          model_ema.update(model, Tensor([projected_ema_decay_val*(i/STEPS)**hyp['ema']['decay_pow']]))
-
+      with Context(BEAM=getenv("LATEBEAM", BEAM.value), WINO=getenv("LATEWINO", WINO.value)):
+        loss = train_step_jitted(model, optim.OptimizerGroup(opt_bias, opt_non_bias), [lr_sched_bias, lr_sched_non_bias], X, Y)
+        et = time.monotonic()
+        loss_cpu = loss.numpy()
+      # EMA for network weights
+      if getenv("EMA") and i > hyp['ema']['steps'] and (i+1) % hyp['ema']['every_n_steps'] == 0:
+        if model_ema is None:
+          model_ema = modelEMA(W, model)
+        model_ema.update(model, Tensor([projected_ema_decay_val*(i/STEPS)**hyp['ema']['decay_pow']]))
       cl = time.monotonic()
-      step_times.append((cl-st)*1000.0)
       device_str = loss.device if isinstance(loss.device, str) else f"{loss.device[0]} * {len(loss.device)}"
       #  53  221.74 ms run,    2.22 ms python,  219.52 ms CL,  803.39 loss, 0.000807 LR, 4.66 GB used,   3042.49 GFLOPS,    674.65 GOPS
       print(f"{i:3d} {(cl-st)*1000.0:7.2f} ms run, {(et-st)*1000.0:7.2f} ms python, {(cl-et)*1000.0:7.2f} ms {device_str}, {loss_cpu:7.2f} loss, {opt_non_bias.lr.numpy()[0]:.6f} LR, {GlobalCounters.mem_used/1e9:.2f} GB used, {GlobalCounters.global_ops*1e-9/(cl-st):9.2f} GFLOPS, {GlobalCounters.global_ops*1e-9:9.2f} GOPS")
       st = cl
       i += 1
 
-  if (assert_time:=getenv("ASSERT_MIN_STEP_TIME")):
-    min_time = min(step_times)
-    assert min_time < assert_time, f"Speed regression, expected min step time of < {assert_time} ms but took: {min_time} ms"
-
   # verify eval acc
   if target := getenv("TARGET_EVAL_ACC_PCT", 0.0):
     if eval_acc_pct >= target:
@@ -433,5 +424,4 @@ def train_cifar():
       raise ValueError(colored(f"{eval_acc_pct=} < {target}", "red"))
 
 if __name__ == "__main__":
-  with WallTimeEvent(BenchEvent.FULL):
-    train_cifar()
+  train_cifar()

examples/llama.py

@@ -13,7 +13,6 @@ from extra.models.llama import Transformer, convert_from_huggingface, fix_bf16
 from sentencepiece import SentencePieceProcessor
 import tiktoken, sys
 from tiktoken.load import load_tiktoken_bpe
-from extra.bench_log import BenchEvent, WallTimeEvent
 
 MAX_CONTEXT = getenv("MAX_CONTEXT", 4096)
 
@@ -207,42 +206,40 @@ class LLaMa:
 
     model = Transformer(**params["args"], linear=linear, max_context=MAX_CONTEXT, jit=bool(JIT))
 
-    with WallTimeEvent(BenchEvent.LOAD_WEIGHTS):
-      if model_path.is_dir():
-        weights = concat_weights([load(filename) for filename in [f"{model_path}/consolidated.{i:02d}.pth" for i in range(params["files"])]], device[0] if isinstance(device, tuple) else device)
-      else:
-        weights = load(str(model_path))
-      if "model.embed_tokens.weight" in weights:
-        weights = convert_from_huggingface(weights, params["args"]["n_layers"], params["args"]["n_heads"], params["args"].get("n_kv_heads", params["args"]["n_heads"]))
+    if model_path.is_dir():
+      weights = concat_weights([load(filename) for filename in [f"{model_path}/consolidated.{i:02d}.pth" for i in range(params["files"])]], device[0] if isinstance(device, tuple) else device)
+    else:
+      weights = load(str(model_path))
+    if "model.embed_tokens.weight" in weights:
+      weights = convert_from_huggingface(weights, model, params["args"]["n_heads"], params["args"].get("n_kv_heads", params["args"]["n_heads"]))
 
-      weights = fix_bf16(weights)
+    weights = fix_bf16(weights)
 
-      # prevent tracking model weights
-      # this is a part of a larger problem with BUFFER UOps and gc in TRACK_MATCH_STATS=2
-      with Context(BEAM=0, TRACK_MATCH_STATS=0):
-        # quantize
-        if quantize is not None:
-          weights = linear.quantize(weights, device)
-          for _,v in weights.items(): v.realize()
+    with Context(BEAM=0):
+      # quantize
+      if quantize is not None:
+        weights = linear.quantize(weights, device)
+        for _,v in weights.items(): v.realize()
 
-        # shard
-        if isinstance(device, tuple):
-          for k,v in nn.state.get_state_dict(model).items():
-            if 'scale' in k: v.shard_(device, axis=None)  # from quantized
-            elif '.attention.' in k:
-              if getenv("SHARD_KVCACHE") and ('.wq.' in k or '.wk.' in k or '.wv.' in k): v.shard_(device, axis=0)
-              else: v.shard_(device, axis=-1)
-            elif '.feed_forward.w1.' in k: v.shard_(device, axis=0)
-            elif '.feed_forward.w3.' in k: v.shard_(device, axis=0)
-            elif '.feed_forward.' in k: v.shard_(device, axis=-1)
-            elif 'tok_embeddings.weight' in k: v.shard_(device, axis=0)
-            elif 'output.weight' in k: v.shard_(device, axis=-1)
-            #elif k.endswith('.weight'): v.shard_(device, axis=-1)
-            #elif 'norm.' in k: v.shard_(device, axis=-1)
-            else: v.shard_(device, axis=None)
+      # shard
+      if isinstance(device, tuple):
+        for k,v in nn.state.get_state_dict(model).items():
+          if 'scale' in k: v.shard_(device, axis=None)  # from quantized
+          elif '.attention.' in k:
+            if getenv("SHARD_KVCACHE") and ('.wq.' in k or '.wk.' in k or '.wv.' in k): v.shard_(device, axis=0)
+            else: v.shard_(device, axis=-1)
+          elif '.feed_forward.w1.' in k: v.shard_(device, axis=0)
+          elif '.feed_forward.w3.' in k: v.shard_(device, axis=0)
+          elif '.feed_forward.' in k: v.shard_(device, axis=-1)
+          elif 'tok_embeddings.weight' in k: v.shard_(device, axis=0)
+          elif 'output.weight' in k: v.shard_(device, axis=-1)
+          #elif k.endswith('.weight'): v.shard_(device, axis=-1)
+          #elif 'norm.' in k: v.shard_(device, axis=-1)
+          else: v.shard_(device, axis=None)
+          #print(k, v.shape, v.lazydata.axis)
 
-        # replace weights in model
-        load_state_dict(model, weights, strict=False, consume=True)
+      # replace weights in model
+      load_state_dict(model, weights, strict=False, consume=True)
 
     return LLaMa(model, tokenizer)
 
@@ -330,6 +327,7 @@ int main()
 \end{code}
 """
 if __name__ == "__main__":
+  Tensor.no_grad = True
   print(f"using {Device.DEFAULT} backend")
 
   parser = argparse.ArgumentParser(description="Run LLaMA in tinygrad", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -445,7 +443,7 @@ After you are done speaking, output [EOS]. You are not Chad.
   print(f"using LLaMA{LLAMA_SUFFIX}-{args.size} model")
   device = tuple(f"{Device.DEFAULT}:{i}" for i in range(args.shard)) if args.shard > 1 else Device.DEFAULT
   llama = LLaMa.build(MODEL_PATH, TOKENIZER_PATH, model_gen=args.gen, model_size=args.size, quantize=args.quantize, device=device)
-  param_bytes = sum(x.uop.size * x.dtype.itemsize for x in get_parameters(llama.model))
+  param_bytes = sum(x.lazydata.size * x.dtype.itemsize for x in get_parameters(llama.model))
 
   outputted = pre_prompt if chatbot else args.prompt
   start_pos, toks = 0, [llama.tokenizer.bos_id()] + llama.tokenizer.encode(outputted)
@@ -477,12 +475,11 @@ After you are done speaking, output [EOS]. You are not Chad.
       next_tok = Tensor([toks[start_pos:]], device=device) if tok_tensor is None or (len(toks)-start_pos) > 1 else tok_tensor.reshape(1, 1)
       with Profiling(enabled=args.profile):
         with Timing("total ", enabled=args.timing, on_exit=lambda x: f", {1e9/x:.2f} tok/s, {GlobalCounters.global_mem/x:.2f} GB/s, param {param_bytes/x:.2f} GB/s"):
-          with WallTimeEvent(BenchEvent.STEP):
-            with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on {Device.DEFAULT}" if DEBUG>=2 else "")+
-                        f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
-                        (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s, param {param_bytes*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None, enabled=args.timing):
-              tok_tensor = llama.model(next_tok, start_pos, args.temperature)
-            tok = tok_tensor.item()
+          with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on GPU" if DEBUG>=2 else "")+
+                      f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
+                      (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s, param {param_bytes*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None, enabled=args.timing):
+            tok_tensor = llama.model(next_tok, start_pos, args.temperature)
+          tok = tok_tensor.item()
 
       # use the kv cache
       start_pos = len(toks)

examples/llama3.py

@@ -7,7 +7,6 @@ from extra.models.llama import Transformer, convert_from_huggingface, convert_fr
 from tinygrad.nn.state import safe_load, torch_load, load_state_dict, get_parameters, gguf_load
 from tinygrad import Tensor, dtypes, nn, Context, Device, GlobalCounters
 from tinygrad.helpers import Profiling, Timing, DEBUG, colored, fetch, tqdm
-from extra.bench_log import BenchEvent, WallTimeEvent
 
 class Tokenizer:
   pat_str = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
@@ -127,8 +126,7 @@ def NF4Linear(block_size):
       return x.linear(unscaled.reshape(self.out_features, self.in_features).T)
 
     @staticmethod
-    def quantize(state_dict: dict[str, Tensor], device, scale_dtype=dtypes.float16, quantize_embeds=False) -> dict[str, Tensor]:
-      assert not quantize_embeds  # TODO: support this?
+    def quantize(state_dict: dict[str, Tensor], device, scale_dtype=dtypes.float16) -> dict[str, Tensor]:
       new_state_dict = {}
       for k, v in state_dict.items():
         if "feed_forward" in k or "attention.w" in k:
@@ -157,11 +155,7 @@ MODEL_PARAMS = {
   "70B": {
     "args": {"dim": 8192, "n_heads": 64, "n_kv_heads": 8, "n_layers": 80, "norm_eps": 1e-5, "rope_theta": 500000, "vocab_size": 128256,  "hidden_dim": 28672},
     "files": 8
-  },
-  "405B": {
-    "args": {"dim": 16384, "n_heads": 128, "n_kv_heads": 8, "n_layers": 126, "norm_eps": 1e-5, "rope_theta": 500000, "vocab_size": 128256,  "hidden_dim": 53248},
-    "files": 191
-  },
+  }
 }
 def build_transformer(model_path: Path, model_size="8B", quantize=None, scale_dtype=dtypes.float16, device=None, max_context=8192, load_weights=True):
   # build model
@@ -171,42 +165,40 @@ def build_transformer(model_path: Path, model_size="8B", quantize=None, scale_dt
   model = Transformer(**MODEL_PARAMS[model_size]["args"], linear=linear, embedding=embedding, max_context=max_context, jit=True)
 
   if not load_weights: return model
-
   # load weights
-  with WallTimeEvent(BenchEvent.LOAD_WEIGHTS):
-    if model_path.is_dir():
-      if (model_path / "model.safetensors.index.json").exists(): weights = load(str(model_path / "model.safetensors.index.json"))
-      elif (model_path / "model.safetensors").exists(): weights = load(str(model_path / "model.safetensors"))
-      else: weights = concat_weights([load(str(model_path / f"consolidated.{i:02d}.pth")) for i in range(MODEL_PARAMS[model_size]["files"])], device[0] if isinstance(device, tuple) else device)
-    else:
-      weights = load(str(model_path))
-    if "model.embed_tokens.weight" in weights:
-      weights = convert_from_huggingface(weights, MODEL_PARAMS[model_size]["args"]["n_layers"], MODEL_PARAMS[model_size]["args"]["n_heads"], MODEL_PARAMS[model_size]["args"]["n_kv_heads"])
-    elif "token_embd.weight" in weights:
-      weights = convert_from_gguf(weights, MODEL_PARAMS[model_size]["args"]["n_layers"])
-    weights = fix_bf16(weights)
+  if model_path.is_dir():
+    if (model_path / "model.safetensors.index.json").exists(): weights = load(str(model_path / "model.safetensors.index.json"))
+    elif (model_path / "model.safetensors").exists(): weights = load(str(model_path / "model.safetensors"))
+    else: weights = concat_weights([load(str(model_path / f"consolidated.{i:02d}.pth")) for i in range(MODEL_PARAMS[model_size]["files"])], device[0] if isinstance(device, tuple) else device)
+  else:
+    weights = load(str(model_path))
+  if "model.embed_tokens.weight" in weights:
+    weights = convert_from_huggingface(weights, model, MODEL_PARAMS[model_size]["args"]["n_heads"], MODEL_PARAMS[model_size]["args"]["n_kv_heads"])
+  elif "token_embd.weight" in weights:
+    weights = convert_from_gguf(weights, model)
+  weights = fix_bf16(weights)
 
-    with Context(BEAM=0):
-      # quantize
-      if quantize == "float16": weights = {k:v.cast(quantize).contiguous() for k,v in weights.items()}
-      elif quantize is not None:
-        weights = linear.quantize(weights, device, scale_dtype, quantize_embeds)
-        for _,v in weights.items(): v.realize()
+  with Context(BEAM=0):
+    # quantize
+    if quantize == "float16": weights = {k:v.cast(quantize).contiguous() for k,v in weights.items()}
+    elif quantize is not None:
+      weights = linear.quantize(weights, device, scale_dtype, quantize_embeds)
+      for _,v in weights.items(): v.realize()
 
-      # shard
-      if isinstance(device, tuple):
-        for k,v in nn.state.get_state_dict(model).items():
-          if 'scale' in k: v.shard_(device, axis=None)  # from quantized
-          elif '.attention.' in k: v.shard_(device, axis=-1)
-          elif '.feed_forward.w1.' in k: v.shard_(device, axis=0)
-          elif '.feed_forward.w3.' in k: v.shard_(device, axis=0)
-          elif '.feed_forward.' in k: v.shard_(device, axis=-1)
-          elif 'tok_embeddings.weight' in k: v.shard_(device, axis=0)
-          elif 'output.weight' in k: v.shard_(device, axis=0)
-          else: v.shard_(device, axis=None)
+    # shard
+    if isinstance(device, tuple):
+      for k,v in nn.state.get_state_dict(model).items():
+        if 'scale' in k: v.shard_(device, axis=None)  # from quantized
+        elif '.attention.' in k: v.shard_(device, axis=-1)
+        elif '.feed_forward.w1.' in k: v.shard_(device, axis=0)
+        elif '.feed_forward.w3.' in k: v.shard_(device, axis=0)
+        elif '.feed_forward.' in k: v.shard_(device, axis=-1)
+        elif 'tok_embeddings.weight' in k: v.shard_(device, axis=0)
+        elif 'output.weight' in k: v.shard_(device, axis=0)
+        else: v.shard_(device, axis=None)
 
-      # replace weights in model
-      load_state_dict(model, weights, strict=False, consume=True)
+    # replace weights in model
+    load_state_dict(model, weights, strict=False, consume=True)
   return model
 
 # default settings
@@ -237,10 +229,12 @@ def prefill(model, toks, start_pos=0):
   return start_pos
 
 if __name__ == "__main__":
+  Tensor.no_grad = True
+
   parser = argparse.ArgumentParser()
   parser.add_argument("--download_model", action="store_true", help="Download a model")
   parser.add_argument("--model", type=Path, help="Model path")
-  parser.add_argument("--size", choices=["1B", "8B", "70B", "405B"], default="1B", help="Model size")
+  parser.add_argument("--size", choices=["1B", "8B", "70B"], default="1B", help="Model size")
   parser.add_argument("--shard", type=int, default=1, help="Shard the model across multiple devices")
   parser.add_argument("--quantize", choices=["int8", "nf4", "float16"], help="Quantization method")
   parser.add_argument("--no_api", action="store_true", help="Disable the api and run a cli test interface")
@@ -248,7 +242,7 @@ if __name__ == "__main__":
   parser.add_argument("--port", type=int, default=7776, help="Web server port")
   parser.add_argument("--debug", action="store_true", help="Enable debug mode")
   parser.add_argument("--seed", type=int, help="Random seed")
-  parser.add_argument("--temperature", type=float, default=0.85, help="Temperature")
+  parser.add_argument("--temperature", type=int, default=0.85, help="Temperature")
   parser.add_argument("--benchmark", action="store_true", help="Run a benchmark")
   parser.add_argument("--timing", action="store_true", help="Print timing per token")
   parser.add_argument("--profile", action="store_true", help="Output profile data")
@@ -288,7 +282,7 @@ if __name__ == "__main__":
 
   device = tuple(f"{Device.DEFAULT}:{i}" for i in range(args.shard)) if args.shard > 1 else Device.DEFAULT
   model = build_transformer(args.model, model_size=args.size, quantize=args.quantize, device=device)
-  param_bytes = sum(x.uop.size * x.dtype.itemsize for x in get_parameters(model))
+  param_bytes = sum(x.lazydata.size * x.dtype.itemsize for x in get_parameters(model))
 
   if not args.no_api and not args.benchmark:
     from bottle import Bottle, request, response, HTTPResponse, abort, static_file
@@ -440,12 +434,11 @@ if __name__ == "__main__":
       st = GlobalCounters.time_sum_s
       with Profiling(enabled=args.profile):
         with Timing("total ", on_exit=lambda x: f", {1e9/x:.2f} tok/s, {GlobalCounters.global_mem/x:.2f} GB/s, param {param_bytes/x:.2f} GB/s"):
-          with WallTimeEvent(BenchEvent.STEP):
-            with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on {Device.DEFAULT}" if DEBUG>=2 else "")+
-                        f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
-                        (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s, param {param_bytes*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None):
-              tok = model(Tensor([[last_tok]], device=device), start_pos, TEMPERATURE, TOP_K, TOP_P, ALPHA_F, ALPHA_P)
-            tok = tok.item()
+          with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on GPU" if DEBUG>=2 else "")+
+                      f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
+                      (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s, param {param_bytes*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None):
+            tok = model(Tensor([[last_tok]], device=device), start_pos, TEMPERATURE, TOP_K, TOP_P, ALPHA_F, ALPHA_P)
+          tok = tok.item()
       start_pos += 1
       last_tok = tok
       generated += tokenizer.decode([tok])
@@ -479,7 +472,7 @@ if __name__ == "__main__":
         st = GlobalCounters.time_sum_s
         with Profiling(enabled=args.profile):
           with Timing("total ", enabled=args.timing, on_exit=lambda x: f", {1e9/x:.2f} tok/s, {GlobalCounters.global_mem/x:.2f} GB/s, param {param_bytes/x:.2f} GB/s"):
-            with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on {Device.DEFAULT}" if DEBUG>=2 else "")+
+            with Timing("enqueue in ", on_exit=(lambda et: (f", {(GlobalCounters.time_sum_s-st)*1e3:.2f} ms on GPU" if DEBUG>=2 else "")+
                         f", {GlobalCounters.global_ops*1e-9:.2f} GOPS, {GlobalCounters.global_mem*1e-9:.2f} GB"+
                         (f", {GlobalCounters.global_mem*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s, param {param_bytes*1e-9/(GlobalCounters.time_sum_s-st):.2f} GB/s" if DEBUG>=2 else "")) if DEBUG else None, enabled=args.timing):
 

examples/llm.c/export.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 import os
 if "NOOPT" not in os.environ: os.environ["NOOPT"] = "1"
-from tinygrad import Device, nn, Tensor, dtypes
+from tinygrad import Device, nn, Tensor, dtypes, Variable
 Device.DEFAULT = "CPU"
 from train_gpt2 import GPT, GPTConfig
-from tinygrad.helpers import dedup, flatten, getenv, GlobalCounters, to_function_name
-from tinygrad.engine.realize import get_kernel
+from tinygrad.helpers import dedup, to_function_name, flatten, getenv, GlobalCounters, ansilen, to_function_name
+from tinygrad.engine.realize import get_kernel, run_schedule
 from tinygrad.engine.memory import memory_planner
-from tinygrad.uop.ops import Ops
+from tinygrad.ops import Ops
 
 TIMING = getenv("TIMING")
 
@@ -16,7 +16,7 @@ if __name__ == "__main__":
   #model.load_pretrained()
   for p in nn.state.get_parameters(model): p.replace(Tensor.empty(p.shape, dtype=p.dtype)) # fake load pretrained
 
-  #early_sched = create_schedule([x.uop for x in nn.state.get_parameters(model)])
+  #early_sched = create_schedule([x.lazydata for x in nn.state.get_parameters(model)])
   #print(f"built model {len(early_sched)}")
 
   #B, T = Variable("B", 1, 128).bind(4), 64 #Variable("T", 1, 1024).bind(64)
@@ -25,7 +25,7 @@ if __name__ == "__main__":
   Tensor.training = True
   optimizer = nn.optim.Adam(nn.state.get_parameters(model), lr=1e-4)
   warmup_count = getenv("WARMUP", 3)
-  for i in range(warmup_count):  # TODO: why does it take three and not two to stabilize
+  for i in range(warmup_count):  # TODO: why does it take three and not two to stablize
     GlobalCounters.reset()
     X = Tensor.empty(4, 64, dtype=dtypes.int).reshape(B, T)
     Y = Tensor.empty(4, 64, dtype=dtypes.int).reshape(B, T)
@@ -56,7 +56,7 @@ if __name__ == "__main__":
   state_dict.update({'X': X, 'Y': Y, 'loss': loss})
   grad_state_dict = {}
   for k,v in state_dict.items():
-    if v.uop.base.buffer not in used_buffers: print(f"UNUSED: {k}")
+    if v.lazydata.base.buffer not in used_buffers: print(f"UNUSED: {k}")
     if v.grad is not None: grad_state_dict['grad_'+k] = v.grad
   state_dict.update(grad_state_dict)
   state_dict.update({'adam_b1_t': optimizer.b1_t, 'adam_b2_t': optimizer.b2_t, 'adam_lr': optimizer.lr})
@@ -65,7 +65,7 @@ if __name__ == "__main__":
     nm = inverse_state_dict[p]
     state_dict["adam_m_"+nm] = m
     state_dict["adam_v_"+nm] = v
-  named_buffers = {v.uop.base.buffer:k.replace(".", "_") for k,v in state_dict.items()}
+  named_buffers = {v.lazydata.base.buffer:k.replace(".", "_") for k,v in state_dict.items()}
 
   c_code = ["#include <stdlib.h>", "#include <tgmath.h>", "#include <stdbool.h>"]
   if TIMING: c_code += ["#include <stdio.h>", "#include <time.h>"]

examples/llm.c/train_gpt2.py

@@ -99,7 +99,7 @@ class GPT:
 
   def __call__(self, idx:Tensor, targets=None):
     b, t = idx.shape
-    pos = Tensor.arange(0, t, device=idx.device)
+    pos = Tensor.arange(0, t)
 
     tok_emb = self.wte(idx) # token embeddings of shape (b, t, n_embd)
     pos_emb = self.wpe(pos) # position embeddings of shape (t, n_embd)
@@ -124,7 +124,6 @@ if __name__ == "__main__":
   parser.add_argument("--batch_size", type=int, default=4, help="batch size")
   parser.add_argument("--sequence_length", type=int, default=64, help="sequence length")
   parser.add_argument("--skip_test", action="store_true", help="skip test")
-  parser.add_argument("--gpus", type=int, default=1, help="sequence length")
   args = parser.parse_args()
   B, T = args.batch_size, args.sequence_length
   assert 1 <= T <= 1024
@@ -132,10 +131,6 @@ if __name__ == "__main__":
   model = GPT(GPTConfig(n_layer=12, n_head=12, n_embd=768))
   model.load_pretrained()
 
-  if args.gpus > 1:
-    GPUS = tuple(f'{Device.DEFAULT}:{i}' for i in range(args.gpus))
-    for x in nn.state.get_parameters(model): x.to_(GPUS)  # we put a copy of the model on every GPU
-
   # init the tokenizer
   enc = tiktoken.get_encoding("gpt2")
   encode = lambda s: enc.encode(s, allowed_special={"<|endoftext|>"})
@@ -170,32 +165,23 @@ if __name__ == "__main__":
   x, y = next(data_iter) # we'll overfit this batch below
   optimizer = nn.optim.AdamW(nn.state.get_parameters(model), lr=1e-4, weight_decay=0)
 
-  print(f"model state:     {sum(x.nbytes() for x in nn.state.get_parameters(model))/1e9:.2f} GB")
-  print(f"optimizer state: {sum(x.nbytes() for x in nn.state.get_parameters(optimizer))/1e9:.2f} GB")
-
-  # shard the data on axis 0
-  if args.gpus > 1: x, y = x.shard(GPUS, axis=0), y.shard(GPUS, axis=0)
-
   @TinyJit
-  @Tensor.train()
-  def step(x:Tensor, y:Tensor) -> Tensor:
+  def step(x, y):
     _, loss = model(x, y)
     optimizer.zero_grad()
     loss.backward()
     return loss.realize(*optimizer.schedule_step())
 
-  for i in range(args.num_iterations):
-    GlobalCounters.reset()
-    t0 = time.perf_counter()
-    loss = step(x.contiguous(), y.contiguous())
-    Device[Device.DEFAULT].synchronize()
-    t1 = time.perf_counter()
-    print(f"iteration {i}, loss: {loss.item():.6f}, time: {(t1-t0)*1000:.3f}ms, {int(B*T/(t1-t0))} tok/s, {GlobalCounters.global_mem/1e9:.2f} GB")
+  with Tensor.train():
+    for i in range(args.num_iterations):
+      GlobalCounters.reset()
+      t0 = time.time()
+      loss = step(x.contiguous(), y.contiguous())
+      Device[Device.DEFAULT].synchronize()
+      t1 = time.time()
+      print(f"iteration {i}, loss: {loss.item():.6f}, time: {(t1-t0)*1000:.3f}ms, {int(B*T/(t1-t0))} tok/s")
 
   if not args.skip_test:
-    # copy back to single gpu for test
-    if args.gpus > 1:
-      for x in nn.state.get_parameters(model): x.to_(Device.DEFAULT)
     start = "<|endoftext|>"
     start_ids = encode(start)
     x = (Tensor(start_ids)[None, ...])

examples/mamba.py

@@ -279,15 +279,9 @@ def generate(model, tokenizer, prompt: str, n_tokens_to_gen: int = 10, temp: boo
   # Loading in the prompt tokens
   logits = model.forward(Tensor([tks]))[:, -1, :]
   for _ in tqdm(range(n_tokens_to_gen), desc="Speed Gen"):
+    # TODO: topk
     if sample:
-      scaled_logits = logits / temp
-      if top_k is not None:
-        topk_values, topk_indices = scaled_logits.topk(top_k)
-        filtered_logits = Tensor.full_like(scaled_logits, -float("inf"))
-        filtered_logits = filtered_logits.scatter(dim=-1, index=topk_indices, src=topk_values)
-        tok_Tens = filtered_logits.softmax().multinomial()
-      else:
-        tok_Tens = scaled_logits.softmax().multinomial()
+      tok_Tens = (logits/temp).softmax().multinomial()
     else:
       tok_Tens = logits.argmax(axis=-1).unsqueeze(0)
     tok = tok_Tens.item()
@@ -304,7 +298,6 @@ if __name__ == "__main__":
   parser.add_argument("--size", type=str, default="370m",
                       help=f"Size of model to use [{', '.join([k for k in MODELS.keys()])}]")
   parser.add_argument("--n_tokens", type=int, default=10, help="Number of tokens to generate")
-  parser.add_argument("--top_k", type=int, help="Limit sampling to the top k most likely tokens")
   parser.add_argument("--sample", dest="sample", action="store_true", help="Sample flag")
   parser.add_argument("--temp", type=float, default=1.0, help="Sampling temp has to be <=1.0")
   args = parser.parse_args()
@@ -315,9 +308,8 @@ if __name__ == "__main__":
   num_toks = args.n_tokens
   sample = args.sample
   temp = args.temp
-  top_k = args.top_k
   s = time.time()
-  tinyoutput = generate(model, tokenizer, prompt, n_tokens_to_gen=num_toks, sample=sample, temp=temp, top_k=top_k)
+  tinyoutput = generate(model, tokenizer, prompt, n_tokens_to_gen=num_toks, sample=sample, temp=temp)
   print(tinyoutput)
   print('TIME: ', time.time() - s)
   TORCHOUTPUT = "Why is gravity \nso important?\nBecause it's the only"

examples/minrf.py

@@ -1,156 +0,0 @@
-# much taken from https://github.com/cloneofsimo/minRF
-from tinygrad import Tensor, nn, GlobalCounters, TinyJit
-from tinygrad.helpers import getenv, trange
-from extra.models.llama import Attention, FeedForward, precompute_freqs_cis
-
-def modulate(x:Tensor, shift:Tensor, scale:Tensor) -> Tensor: return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1)
-
-# TODO: why doesn't the TimestepEmbedder from minRF work?
-class TimestepEmbedder:
-  def __init__(self, hidden_size): self.mlp = [nn.Linear(1, hidden_size), Tensor.silu, nn.Linear(hidden_size, hidden_size)]
-  def __call__(self, t:Tensor): return t.reshape(-1, 1).sequential(self.mlp)
-
-class TransformerBlock:
-  def __init__(self, dim, n_heads, norm_eps=1e-5):
-    self.attention = Attention(dim, n_heads)
-    self.feed_forward = FeedForward(dim, 4*dim)
-    self.attention_norm = nn.LayerNorm(dim, eps=norm_eps)
-    self.ffn_norm = nn.LayerNorm(dim, eps=norm_eps)
-    self.adaLN_modulation = nn.Linear(dim, 6 * dim, bias=True)
-
-  def __call__(self, x:Tensor, freqs_cis:Tensor, adaln_input:Tensor):
-    shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input.silu()).chunk(6, dim=1)
-    x = x + gate_msa.unsqueeze(1) * self.attention(modulate(self.attention_norm(x), shift_msa, scale_msa), 0, freqs_cis)
-    x = x + gate_mlp.unsqueeze(1) * self.feed_forward(modulate(self.ffn_norm(x), shift_mlp, scale_mlp))
-    return x.contiguous().contiguous_backward()
-
-class FinalLayer:
-  def __init__(self, dim, patch_size, out_channels):
-    self.norm_final = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
-    self.linear = nn.Linear(dim, patch_size*patch_size*out_channels, bias=True)
-    self.adaLN_modulation = nn.Linear(dim, 2 * dim, bias=True)
-
-    # init weights/bias to 0
-    self.linear.weight.replace(self.linear.weight.zeros_like().contiguous())
-    self.linear.bias.replace(self.linear.bias.zeros_like().contiguous())
-
-  def __call__(self, x:Tensor, c:Tensor):
-    shift, scale = self.adaLN_modulation(c.silu()).chunk(2, dim=1)
-    x = modulate(self.norm_final(x), shift, scale)
-    return self.linear(x)
-
-# channels=1, input_size=32, dim=64, n_layers=6, n_heads=4, num_classes=10
-class DiT_Llama:
-  def __init__(self, in_channels=1, dim=64, n_layers=6, n_heads=4, num_classes=10, patch_size=2):
-    self.patch_size = patch_size
-    self.out_channels = in_channels
-    self.num_classes = num_classes
-
-    self.init_conv_seq = [
-      nn.Conv2d(in_channels, dim // 2, kernel_size=5, padding=2, stride=1), Tensor.silu, nn.GroupNorm(32, dim//2),
-      nn.Conv2d(dim //2, dim // 2, kernel_size=5, padding=2, stride=1), Tensor.silu, nn.GroupNorm(32, dim//2),
-    ]
-
-    self.x_embedder = nn.Linear(self.patch_size * self.patch_size * dim // 2, dim, bias=True)
-    self.t_embedder = TimestepEmbedder(dim)
-    self.y_embedder = nn.Embedding(num_classes+1, dim)
-    self.final_layer = FinalLayer(dim, self.patch_size, self.out_channels)
-
-    self.freqs_cis = precompute_freqs_cis(dim // n_heads, 4096)
-    self.layers = [TransformerBlock(dim, n_heads) for _ in range(n_layers)]
-
-  def unpatchify(self, x:Tensor):
-    c, p = self.out_channels, self.patch_size
-    h = w = int(x.shape[1] ** 0.5)
-    x = x.reshape(shape=(x.shape[0], h, w, p, p, c))
-    x = x.rearrange("n h w p q c -> n c h p w q")
-    return x.reshape(shape=(x.shape[0], c, h * p, h * p))
-
-  def patchify(self, x:Tensor):
-    B, C, H, W = x.shape
-    x = x.reshape(B, C, H // self.patch_size, self.patch_size, W // self.patch_size, self.patch_size)
-    x = x.permute(0, 2, 4, 1, 3, 5).flatten(-3).flatten(1, 2)
-    return x  # B <H*W ish> <C*patch_size*patch_size>
-
-  def __call__(self, x:Tensor, t:Tensor, y:Tensor) -> Tensor:
-    x = x.sequential(self.init_conv_seq)
-    x = self.patchify(x)
-    x = self.x_embedder(x)
-    adaln_input = self.t_embedder(t) + self.y_embedder(y)
-    adaln_input = adaln_input.contiguous()
-    for layer in self.layers:
-      x = layer(x, self.freqs_cis[:, :x.size(1)], adaln_input=adaln_input)
-    x = self.final_layer(x, adaln_input)
-    return self.unpatchify(x)
-
-  def rf(self, x:Tensor, cond:Tensor):
-    b = x.shape[0]
-    # self.ln is True
-    t = Tensor.randn((b,)).sigmoid()
-    texp = t.view([b, *([1] * len(x.shape[1:]))])
-
-    # conditional dropout
-    dropout_prob = 0.1
-    cond = (Tensor.rand(cond.shape[0]) < dropout_prob).where(cond.full_like(self.num_classes), cond)
-
-    # this is rectified flow
-    z1 = x.randn_like()
-    zt = (1 - texp) * x + texp * z1
-    vtheta = self(zt, t, cond)
-
-    # MSE loss
-    return ((z1 - x) - vtheta).square().mean()
-
-  def sample(self, z, cond, null_cond, sample_steps=50, cfg=2.0):
-    b = z.size(0)
-    dt = Tensor.full((b,)+(1,)*len(z.shape[1:]), fill_value=1.0/sample_steps).contiguous()
-    images = [z]
-    for i in range(sample_steps, 0, -1):
-      t = Tensor.full((b,), fill_value=i/sample_steps).contiguous()
-      vc = self(z, t, cond)
-      vu = self(z, t, null_cond)
-      vc = vu + cfg * (vc - vu)
-      z = z - dt * vc
-      z = z.contiguous()
-      images.append(z)
-    return images
-
-def mviz(t:Tensor):
-  assert len(t.shape) == 4 and t.shape[1] == 1
-  ft = t.permute(1,2,0,3).reshape(32, -1)
-  assert ft.shape[-1]%32 == 0
-  print("")
-  for y in ((ft+1)/2).clamp(0,1).tolist():
-    ln = [f"\033[38;5;{232+int(x*23)}m██" for x in y]
-    print(''.join(ln) + "\033[0m")
-
-if __name__ == "__main__":
-  X_train, Y_train, X_test, Y_test = nn.datasets.mnist()
-  X_train = X_train.pad((2,2,2,2))
-  X_train = ((X_train.float()/255)-0.5)/0.5
-  Y_train = Y_train.int()
-
-  model = DiT_Llama(patch_size=getenv("PATCH_SIZE", 2))
-  for r in nn.state.get_parameters(model): r.realize()
-  optimizer = nn.optim.Adam(nn.state.get_parameters(model), lr=5e-4)
-
-  @TinyJit
-  @Tensor.train()
-  def train_step():
-    if getenv("OVERFIT"): samples = Tensor.zeros(getenv("BS", 256), dtype='int')
-    else: samples = Tensor.randint(getenv("BS", 256), high=X_train.shape[0])
-    optimizer.zero_grad()
-    loss = model.rf(X_train[samples], Y_train[samples])
-    loss.backward()
-    optimizer.step()
-    return loss
-
-  @TinyJit
-  def sample(z:Tensor, cond:Tensor) -> Tensor:
-    return model.sample(z, cond, Tensor.full_like(cond, 10), sample_steps=getenv("SAMPLE_STEPS", 20))[-1]
-
-  for steps in (t:=trange(getenv("STEPS", 5000))):
-    if steps%10 == 0: mviz(sample(Tensor.randn(3, 1, 32, 32), Tensor([5,0,4], dtype='int')))
-    GlobalCounters.reset()
-    loss = train_step()
-    t.set_description(f"loss: {loss.item():9.2f}")

examples/mixtral.py

@@ -3,7 +3,6 @@ from tinygrad import Tensor, nn, Device, GlobalCounters, Variable
 from tinygrad.helpers import Timing, Profiling, CI, tqdm
 from tinygrad.nn.state import torch_load, get_state_dict
 from extra.models.llama import FeedForward, Transformer
-from extra.bench_log import BenchEvent, WallTimeEvent
 
 class MixtureFeedForward:
   def __init__(self, num_experts:int, dim:int, hidden_dim:int, linear=nn.Linear):
@@ -31,19 +30,18 @@ if __name__ == "__main__":
                       help="Path to the downloaded weights")
   args = parser.parse_args()
 
-  with WallTimeEvent(BenchEvent.LOAD_WEIGHTS):
-    state = torch_load(args.weights + "/consolidated.00.pth.b")
-    model = Transformer(n_layers=32, dim=4096, hidden_dim=14336, n_heads=32, n_kv_heads=8, norm_eps=1e-5, vocab_size=32000, feed_forward=functools.partial(MixtureFeedForward, 8), jit=False)
-    model_state_dict = get_state_dict(model)
+  state = torch_load(args.weights + "/consolidated.00.pth.b")
+  model = Transformer(n_layers=32, dim=4096, hidden_dim=14336, n_heads=32, n_kv_heads=8, norm_eps=1e-5, vocab_size=32000, feed_forward=functools.partial(MixtureFeedForward, 8), jit=False)
+  model_state_dict = get_state_dict(model)
 
-    for k in (t := tqdm(state, disable=CI)):
-      if 'feed_forward.experts.' in k:
-        expert_no = int(k.split('feed_forward.experts.')[1].split('.')[0])
-        device = Device.DEFAULT + ":" + str((expert_no//2)+1)
-      else:
-        device = Device.DEFAULT
-      t.set_description(f"ram used: {GlobalCounters.mem_used/1e9:5.2f} GB, loading {k} to {device}")
-      model_state_dict[k].replace(state[k].to(device).half()).realize()
+  for k in (t := tqdm(state, disable=CI)):
+    if 'feed_forward.experts.' in k:
+      expert_no = int(k.split('feed_forward.experts.')[1].split('.')[0])
+      device = Device.DEFAULT + ":" + str((expert_no//2)+1)
+    else:
+      device = Device.DEFAULT
+    t.set_description(f"ram used: {GlobalCounters.mem_used/1e9:5.2f} GB, loading {k} to {device}")
+    model_state_dict[k].replace(state[k].to(device).half()).realize()
   if CI: print(f"ram used: {GlobalCounters.mem_used/1e9:5.2f} GB")
 
   from sentencepiece import SentencePieceProcessor
@@ -55,8 +53,7 @@ if __name__ == "__main__":
     GlobalCounters.reset()
     with Profiling(sort="time", frac=0.1, enabled=args.profile):
       with Timing("total ", enabled=args.timing, on_exit=lambda x: f", {1e9/x:.2f} tok/sec"):
-        with WallTimeEvent(BenchEvent.STEP):
-          tok = model(Tensor([toks[start_pos:]]), 0 if start_pos == 0 else Variable("start_pos", 1, 1024-1).bind(start_pos), args.temperature).item()
+        tok = model(Tensor([toks[start_pos:]]), 0 if start_pos == 0 else Variable("start_pos", 1, 1024).bind(start_pos), args.temperature).item()
     toks.append(tok)
     start_pos += 1
     print(spp.decode(toks))

examples/mlperf/dataloader.py

@@ -1,12 +1,11 @@
-import os, random, pickle, queue, struct, math, functools, hashlib, time
+import os, random, pickle, queue
 from typing import List
 from pathlib import Path
 from multiprocessing import Queue, Process, shared_memory, connection, Lock, cpu_count
 
 import numpy as np
 from tinygrad import dtypes, Tensor
-from tinygrad.helpers import getenv, prod, Context, round_up, tqdm, OSX
-from tinygrad.nn.state import TensorIO
+from tinygrad.helpers import getenv, prod, Context, round_up, tqdm
 
 ### ResNet
 
@@ -72,7 +71,7 @@ def loader_process(q_in, q_out, X:Tensor, seed):
       #storage_tensor._copyin(img_tensor.numpy())
 
       # faster
-      X[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
+      X[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
 
       # ideal
       #X[idx].assign(img.tobytes())   # NOTE: this is slow!
@@ -130,15 +129,14 @@ def batch_load_resnet(batch_size=64, val=False, shuffle=True, seed=None, pad_fir
   q_in, q_out = Queue(), Queue()
 
   sz = (batch_size*BATCH_COUNT, 224, 224, 3)
-  shm_name = "resnet_X_val" if val else "resnet_X_train"
-  if not OSX and os.path.exists(f"/dev/shm/{shm_name}"): os.unlink(f"/dev/shm/{shm_name}")
-  shm = shared_memory.SharedMemory(name=shm_name, create=True, size=prod(sz))
+  if os.path.exists("/dev/shm/resnet_X"): os.unlink("/dev/shm/resnet_X")
+  shm = shared_memory.SharedMemory(name="resnet_X", create=True, size=prod(sz))
   procs = []
 
   try:
     # disk:shm is slower
-    if OSX: X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:shm:{shm.name}")
-    else: X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/{shm_name}")
+    #X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:shm:{shm.name}")
+    X = Tensor.empty(*sz, dtype=dtypes.uint8, device=f"disk:/dev/shm/resnet_X")
     Y = [None] * (batch_size*BATCH_COUNT)
 
     for _ in range(cpu_count()):
@@ -263,8 +261,8 @@ def load_unet3d_data(preprocessed_dataset_dir, seed, queue_in, queue_out, X:Tens
       x = random_brightness_augmentation(x)
       x = gaussian_noise(x)
 
-    X[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = x.tobytes()
-    Y[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = y.tobytes()
+    X[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = x.tobytes()
+    Y[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = y.tobytes()
 
     queue_out.put(idx)
   queue_out.put(None)
@@ -314,7 +312,7 @@ def batch_load_unet3d(preprocessed_dataset_dir:Path, batch_size:int=6, val:bool=
       proc = Process(target=load_unet3d_data, args=(preprocessed_dataset_dir, seed, queue_in, queue_out, X, Y))
       proc.daemon = True
       proc.start()
-
+      
       procs.append(proc)
 
     for bc in range(batch_count):
@@ -378,12 +376,12 @@ def load_retinanet_data(base_dir:Path, val:bool, queue_in:Queue, queue_out:Queue
       clipped_match_idxs = np.clip(match_idxs, 0, None)
       clipped_boxes, clipped_labels = tgt["boxes"][clipped_match_idxs], tgt["labels"][clipped_match_idxs]
 
-      boxes[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_boxes.tobytes()
-      labels[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_labels.tobytes()
-      matches[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = match_idxs.tobytes()
-      anchors[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = anchor.tobytes()
+      boxes[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_boxes.tobytes()
+      labels[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_labels.tobytes()
+      matches[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = match_idxs.tobytes()
+      anchors[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = anchor.tobytes()
 
-    imgs[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
+    imgs[idx].contiguous().realize().lazydata.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
 
     queue_out.put(idx)
   queue_out.put(None)
@@ -511,301 +509,6 @@ def batch_load_retinanet(dataset, val:bool, base_dir:Path, batch_size:int=32, sh
       # happens with BENCHMARK set
       pass
 
-# stable diffusion callbacks to match mlperf ref; declared here because they're pickled
-def filter_dataset(sample:dict): return {k:v for k,v in sample.items() if k in {'npy', 'txt'}}
-def collate(batch:list[dict]):
-  ret = {"npy": [], "txt": [], "__key__": []}
-  for sample in batch:
-    for k,v in sample.items():
-      ret[k].append(v)
-  return ret
-def collate_fn(batch): return batch
-
-# Reference (code): https://github.com/mlcommons/training/blob/2f4a93fb4888180755a8ef55f4b977ef8f60a89e/stable_diffusion/ldm/data/webdatasets.py, Line 55
-# Reference (params): https://github.com/mlcommons/training/blob/ab4ae1ca718d7fe62c369710a316dff18768d04b/stable_diffusion/configs/train_01x08x08.yaml, Line 107
-def batch_load_train_stable_diffusion(urls:str, BS:int):
-  import webdataset
-  dataset = webdataset.WebDataset(urls=urls, resampled=True, cache_size=-1, cache_dir=None)
-  dataset = dataset.shuffle(size=1000)
-  dataset = dataset.decode()
-  dataset = dataset.map(filter_dataset)
-  dataset = dataset.batched(BS, partial=False, collation_fn=collate)
-  dataset = webdataset.WebLoader(dataset, batch_size=None, shuffle=False, num_workers=1, persistent_workers=True, collate_fn=collate_fn)
-
-  for x in dataset:
-    assert isinstance(x, dict) and all(isinstance(k, str) for k in x.keys()) and all(isinstance(v, list) for v in x.values())
-    assert all(isinstance(moment_mean_logvar, np.ndarray) and moment_mean_logvar.shape==(1,8,64,64) for moment_mean_logvar in x["npy"])
-    assert all(isinstance(caption, str) for caption in x["txt"])
-    yield x
-
-# llama3
-
-class BinIdxDataset:
-  def __init__(self, base_path:Path):
-    self.idx_t = Tensor(base_path.with_name(f"{base_path.name}.idx"))
-    self.idx = TensorIO(self.idx_t)
-
-    # parse idx file
-    magic = self.idx.read(9)
-    assert magic == b"MMIDIDX\x00\x00", "invalid index file format"
-    version, = struct.unpack("<Q", self.idx.read(8))
-    assert version == 1, "unsupported index version"
-    dtype_code, = struct.unpack("<B", self.idx.read(1))
-    self.dtype = {1:dtypes.uint8, 2:dtypes.int8, 3:dtypes.int16, 4:dtypes.int32, 5:dtypes.int64, 6:dtypes.float64, 7:dtypes.double, 8:dtypes.uint16}[dtype_code]
-    self.count, = struct.unpack("<Q", self.idx.read(8))
-    doc_count, = struct.unpack("<Q", self.idx.read(8))
-
-    start = self.idx.tell()
-    end = start + self.count * dtypes.int32.itemsize
-    self.sizes = self.idx_t[start:end].bitcast(dtypes.int32).numpy()
-
-    start = end
-    end = start + self.count * dtypes.int64.itemsize
-    self.pointers = self.idx_t[start:end].bitcast(dtypes.int64).numpy()
-
-    start = end
-    end = start + doc_count * dtypes.int64.itemsize
-    self.doc_idx = self.idx_t[start:end].bitcast(dtypes.int64).numpy()
-
-    # bin file
-    self.bin_t = Tensor(base_path.with_name(f"{base_path.name}.bin"))
-
-  def _index(self, idx) -> tuple[int, int]:
-    return int(self.pointers[idx]), int(self.sizes[idx])
-
-  def get(self, idx, offset:int=0, length:int|None=None):
-    ptr, size = self._index(idx)
-    if length is None: length = size - offset
-    ptr += offset * self.dtype.itemsize
-    return self.bin_t[ptr:ptr+length*self.dtype.itemsize].bitcast(self.dtype).to(None)
-
-# https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/datasets.html
-class GPTDataset:
-  def __init__(self, base_path:Path, samples:int, seqlen:int, seed:int, shuffle:bool):
-    self.samples, self.seqlen = samples, seqlen
-    self.shuffle = shuffle
-    self.rng = np.random.RandomState(seed)
-
-    self.indexed_dataset = BinIdxDataset(base_path)
-
-    # check for cache
-    cache_hash = hashlib.sha256(f"{samples}:{seqlen}:{seed}:{shuffle}".encode()).hexdigest()
-    cache_path = base_path.with_name(f"{base_path.name}.{cache_hash}.index_cache")
-    print(f"try loading GPTDataset from {cache_path}...")
-    if cache_path.exists():
-      print("cache found, loading...")
-      with open(cache_path, "rb") as f:
-        self.doc_idx, self.sample_idx, self.shuffle_idx = pickle.load(f)
-    else:
-      print("cache not found, building index...")
-      self.doc_idx = self._build_doc_idx()
-      self.sample_idx = self._build_sample_idx()
-      self.shuffle_idx = self._build_shuffle_idx()
-      # save cache
-      with open(cache_path, "wb") as f:
-        pickle.dump((self.doc_idx, self.sample_idx, self.shuffle_idx), f)
-
-  def __getitem__(self, idx):
-    if idx is None:
-      text = self._get(0)
-    else:
-      text = self._get(idx)
-
-    return text
-
-  def _get(self, idx):
-    idx = self.shuffle_idx[idx]
-
-    doc_idx_beg, doc_idx_beg_offset = self.sample_idx[idx]
-    doc_idx_end, doc_idx_end_offset = self.sample_idx[idx + 1]
-
-    doc_ids, sample_parts = [], []
-
-    if doc_idx_beg == doc_idx_end:
-      doc_ids.append(self.doc_idx[doc_idx_beg])
-
-      sample_parts.append(
-          self.indexed_dataset.get(
-            int(self.doc_idx[doc_idx_beg]), offset=int(doc_idx_beg_offset), length=int(doc_idx_end_offset - doc_idx_beg_offset + 1)))
-    else:
-      for i in range(doc_idx_beg, doc_idx_end + 1):
-        doc_ids.append(self.doc_idx[i])
-
-        offset = 0 if i > doc_idx_beg else doc_idx_beg_offset
-        length = None if i < doc_idx_end else int(doc_idx_end_offset + 1)
-        sample_parts.append(self.indexed_dataset.get(int(self.doc_idx[i]), offset=int(offset), length=length))
-
-    # concat all parts
-    text = Tensor.cat(*sample_parts)
-
-    return text
-
-  @functools.cached_property
-  def tokens_per_epoch(self) -> int:
-    return sum(self.indexed_dataset.sizes.tolist())
-
-  @functools.cached_property
-  def num_epochs(self) -> int:
-    # we need enough epochs to cover the requested amount of tokens
-    num_epochs = 1
-    num_tokens = self.tokens_per_epoch
-    while num_tokens < self.samples * self.seqlen:
-      num_epochs += 1
-      num_tokens += self.tokens_per_epoch
-    return num_epochs
-
-  # https://github.com/NVIDIA/Megatron-LM/blob/94bd476bd840c2fd4c3ebfc7448c2af220f4832b/megatron/core/datasets/gpt_dataset.py#L558
-  def _build_doc_idx(self):
-    print(f"building doc_idx for {self.num_epochs=}, {self.indexed_dataset.count=}")
-    st = time.perf_counter()
-    # doc_idx = np.mgrid[:self.num_epochs, :self.indexed_dataset.count][1]
-    doc_idx = np.arange(self.indexed_dataset.count).reshape(1, -1).repeat(self.num_epochs, axis=0).flatten()
-    doc_idx = doc_idx.astype(np.int32)
-    at = time.perf_counter()
-    if self.shuffle: self.rng.shuffle(doc_idx)
-    print(f"doc_idx built in {at - st:.3f}s, shuffled in {time.perf_counter() - at:.3f}s")
-    return doc_idx
-
-  def _build_sample_idx(self):
-    print(f"building sample_idx for {self.samples=}, {self.seqlen=}, {self.doc_idx.shape[0]=}")
-    sample_idx_max = max(self.doc_idx.shape[0], self.indexed_dataset.sizes.max())
-    sample_idx = np.empty((self.samples + 1, 2), dtype=np.int64 if sample_idx_max > dtypes.int32.max else np.int32)
-
-    sample_idx_idx, doc_idx_idx, doc_offset = 0, 0, 0
-    sample_idx[sample_idx_idx, 0], sample_idx[sample_idx_idx, 1] = doc_idx_idx, doc_offset
-    sample_idx_idx += 1
-
-    for _ in tqdm(range(1, self.samples + 1)):
-      remaining_seqlen = self.seqlen + 1
-      while remaining_seqlen > 0:
-        doc_idx = int(self.doc_idx[doc_idx_idx])
-        doc_len = int(self.indexed_dataset.sizes[doc_idx]) - doc_offset
-        remaining_seqlen -= doc_len
-        if remaining_seqlen <= 0:
-          doc_offset += remaining_seqlen + doc_len - 1
-          remaining_seqlen = 0
-        else:
-          if doc_idx_idx == len(self.doc_idx) - 1:
-            assert sample_idx_idx == self.samples
-            doc_idx = int(self.doc_idx[doc_idx_idx])
-            doc_offset = int(self.indexed_dataset.sizes[doc_idx]) - 1
-            break
-          doc_idx_idx += 1
-          doc_offset = 0
-
-      sample_idx[sample_idx_idx, 0], sample_idx[sample_idx_idx, 1] = doc_idx_idx, doc_offset
-      sample_idx_idx += 1
-
-    return sample_idx
-
-  def _build_shuffle_idx(self):
-    print(f"building shuffle_idx for {self.samples=}")
-    st = time.perf_counter()
-    shuffle_idx = np.arange(self.samples, dtype=np.int32)
-    at = time.perf_counter()
-    if self.shuffle: self.rng.shuffle(shuffle_idx)
-    print(f"shuffle_idx built in {at - st:.3f}s, shuffled in {time.perf_counter() - at:.3f}s")
-    return shuffle_idx
-
-class BlendedGPTDataset:
-  def __init__(self, paths:list[Path], weights:list[float], samples:int, seqlen:int, seed:int, shuffle:bool):
-    self.shuffle = shuffle
-    self.rng = np.random.RandomState(seed)
-
-    # normalize weights
-    total_weight = sum(weights)
-    self.weights = [w / total_weight for w in weights]
-
-    self.samples = samples
-    surplus = 0.005
-    samples_per_blend = [math.ceil(math.ceil(self.samples * w) * (1 + surplus)) for w in self.weights]
-
-    self.datasets = [GPTDataset(path, samples_per_blend[i], seqlen, seed + i, shuffle) for i,path in enumerate(paths)]
-
-    # check for cache
-    cache_hash = hashlib.sha256(f"{samples}:{seqlen}:{seed}:{shuffle}".encode()).hexdigest()
-    cache_path = paths[0].with_name(f"{paths[0].name}.{cache_hash}.blend_cache")
-    print(f"try loading BlendedGPTDataset from {cache_path}...")
-    if cache_path.exists():
-      print("cache found, loading...")
-      with open(cache_path, "rb") as f:
-        self.dataset_idx, self.dataset_sample_idx = pickle.load(f)
-    else:
-      print("cache not found, building index...")
-      self.dataset_idx, self.dataset_sample_idx = self._build_blend_idx()
-      # save cache
-      with open(cache_path, "wb") as f:
-        pickle.dump((self.dataset_idx, self.dataset_sample_idx), f)
-
-  def get(self, idx:int):
-    tokens = self.datasets[self.dataset_idx[idx]][self.dataset_sample_idx[idx]]
-    return tokens
-
-  def _build_blend_idx(self):
-    dataset_idx = np.zeros(self.samples, dtype=np.int16)
-    dataset_sample_idx = np.zeros(self.samples, dtype=np.int64)
-
-    unspent_datasets = set(range(len(self.datasets)))
-    dataset_sample_counts = [0] * len(self.datasets)
-
-    for i in tqdm(range(self.samples)):
-      error_argmax, error_max = 0, 0.0
-      for di in unspent_datasets:
-        error = self.weights[di] * max(i, 1) - dataset_sample_counts[di]
-        if error > error_max:
-          error_max = error
-          error_argmax = di
-
-      dataset_idx[i] = error_argmax
-      dataset_sample_idx[i] = dataset_sample_counts[error_argmax]
-
-      dataset_sample_counts[error_argmax] += 1
-
-    return dataset_idx, dataset_sample_idx
-
-def batch_load_llama3(bs:int, samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True):
-  if val:
-    dataset = BlendedGPTDataset([
-      base_dir / "validation" / "c4-validationn-91205-samples.en_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, False)
-  else:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-train.en_6_text_document",
-      base_dir / "c4-train.en_7_text_document",
-    ], [
-      1.0, 1.0
-    ], samples, seqlen, seed, True)
-
-  for b in range(math.ceil(samples / bs)):
-    batch = []
-    for i in range(bs):
-      tokens = dataset.get(b * bs + i)
-      batch.append(tokens)
-    yield Tensor.stack(batch, dim=0)
-
-def batch_load_llama3_small(bs:int, samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True):
-  if val:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-validation-91205-samples.en_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, False)
-  else:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-train.en_6_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, True)
-
-  for b in range(math.ceil(samples / bs)):
-    batch = []
-    for i in range(bs):
-      tokens = dataset.get(b * bs + i)
-      batch.append(tokens)
-    yield Tensor.stack(batch, dim=0)
-
 if __name__ == "__main__":
   def load_unet3d(val):
     assert not val, "validation set is not supported due to different sizes on inputs"
@@ -829,23 +532,11 @@ if __name__ == "__main__":
   def load_retinanet(val):
     from extra.datasets.openimages import BASEDIR, download_dataset
     from pycocotools.coco import COCO
-    dataset = COCO(download_dataset(base_dir:=getenv("BASEDIR", BASEDIR), "validation" if val else "train"))
+    dataset = COCO(download_dataset(base_dir:=getenv("BASE_DIR", BASEDIR), "validation" if val else "train"))
     with tqdm(total=len(dataset.imgs.keys())) as pbar:
       for x in batch_load_retinanet(dataset, val, base_dir):
         pbar.update(x[0].shape[0])
 
-  def load_llama3(val):
-    bs = 24
-    samples = 5760 if val else 1_200_000 * 1152
-    seqlen = 8192
-
-    max_, min_ = 0, math.inf
-    for tokens in tqdm(batch_load_llama3(bs, samples, seqlen, Path(getenv("BASEDIR", "/raid/datasets/c4/")), seed=5760, val=bool(val)), total=samples//bs):
-      max_ = max(max_, tokens.shape[1])
-      min_ = min(min_, tokens.shape[1])
-    print(f"max seq length: {max_}")
-    print(f"min seq length: {min_}")
-
   load_fn_name = f"load_{getenv('MODEL', 'resnet')}"
   if load_fn_name in globals():
     globals()[load_fn_name](getenv("VAL", 1))

examples/mlperf/helpers.py

@@ -212,7 +212,7 @@ def get_mlperf_bert_model():
   from examples.mlperf.initializers import LinearBert, EmbeddingBert, LayerNormBert
 
   bert.Linear = LinearBert
-  bert.Embedding = EmbeddingBert
+  bert.Embedding = EmbeddingBert 
   bert.LayerNorm = LayerNormBert
 
   from extra.models.bert import BertForPretraining

examples/mlperf/initializers.py

@@ -2,9 +2,7 @@ import math
 from typing import Union
 
 from tinygrad import Tensor, nn, dtypes
-from tinygrad.helpers import prod, argfix, Context
-from tinygrad.nn.state import get_parameters
-from extra.models.unet import UNetModel
+from tinygrad.helpers import prod, argfix
 
 # rejection sampling truncated randn
 def rand_truncn(*shape, dtype=None, truncstds=2, **kwargs) -> Tensor:
@@ -19,10 +17,6 @@ def he_normal(*shape, a: float = 0.00, **kwargs) -> Tensor:
   std = math.sqrt(2.0 / (1 + a ** 2)) / math.sqrt(prod(argfix(*shape)[1:])) / 0.87962566103423978
   return std * rand_truncn(*shape, **kwargs)
 
-# Stable Diffusion v2 training uses default torch gelu, which doesn't use tanh approximation
-def gelu_erf(x:Tensor) -> Tensor:
-  return 0.5 * x * (1.0 + (x / 1.4142135623730951).erf())
-
 class Conv2dHeNormal(nn.Conv2d):
   def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True):
     super().__init__(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups, bias=bias)
@@ -45,7 +39,7 @@ class LinearBert(nn.Linear):
   def __init__(self, in_features, out_features, bias=True, std=0.02):
     self.weight = std * rand_truncn(out_features, in_features, dtype=dtypes.float32)
     self.bias = Tensor.zeros(out_features, dtype=dtypes.float32) if bias else None
-
+  
   def __call__(self, x:Tensor):
     return x.cast(dtypes.default_float).linear(self.weight.cast(dtypes.default_float).transpose(), self.bias.cast(dtypes.default_float) if self.bias is not None else None)
 
@@ -133,59 +127,3 @@ class Conv2dRetinaNet(nn.Conv2d):
   def __call__(self, x:Tensor) -> Tensor:
     return x.conv2d(self.weight.cast(dtypes.default_float), self.bias.cast(dtypes.default_float) if self.bias is not None else None,
                     groups=self.groups, stride=self.stride, dilation=self.dilation, padding=self.padding)
-
-# copy torch AMP: isolate mixed precision to just the below autocast ops, instead of using dtypes.default_float which affects all new Tensors
-class AutocastLinear(nn.Linear):
-  cast_dtype=dtypes.bfloat16 # enable monkeypatching of the mixed precision dtype
-  def __call__(self, x:Tensor) -> Tensor:
-    dtype = type(self).cast_dtype
-    return x.cast(dtype).linear(self.weight.cast(dtype).transpose(), self.bias.cast(dtype) if self.bias is not None else None)
-
-class AutocastConv2d(nn.Conv2d):
-  cast_dtype=dtypes.bfloat16
-  def __call__(self, x:Tensor) -> Tensor:
-    dtype = type(self).cast_dtype
-    return x.cast(dtype).conv2d(self.weight.cast(dtype), self.bias.cast(dtype), self.groups, self.stride, self.dilation, self.padding)
-
-# copy torch AMP: upcast to float32 before GroupNorm and LayerNorm
-class AutocastGroupNorm(nn.GroupNorm):
-  def __call__(self, x:Tensor) -> Tensor:
-    return super().__call__(x.cast(dtypes.float32))
-
-class AutocastLayerNorm(nn.LayerNorm):
-  def __call__(self, x:Tensor) -> Tensor:
-    return super().__call__(x.cast(dtypes.float32))
-
-def zero_module(module):
-  for p in get_parameters(module): p.assign(Tensor.zeros_like(p).contiguous())
-
-# Stable Diffusion mlperf reference doesn't call scaled_dot_product_attention
-# copy torch AMP: upcast to float32 before softmax on CUDA
-def attn_f32_softmax(q:Tensor, k:Tensor, v:Tensor) -> Tensor:
-  return (q.matmul(k.transpose(-2,-1), dtype=dtypes.float32) / math.sqrt(q.shape[-1])).softmax(-1).cast(q.dtype) @ v
-
-def init_stable_diffusion(version:str, pretrained:str, devices:list[str]):
-  from examples.stable_diffusion import StableDiffusion
-  from tinygrad.nn.state import safe_load, safe_save, load_state_dict, get_state_dict
-  from tempfile import TemporaryDirectory
-  model = StableDiffusion(version=version, pretrained=pretrained)
-  unet:UNetModel = model.model.diffusion_model
-
-  # this prevents extra consumption of memory, enabling much larger BS
-  Tensor.realize(*get_parameters(unet))
-  with TemporaryDirectory(prefix="unet_init") as tmp:
-    safe_save(get_state_dict(unet), init_fn:=f"{tmp}/init_model.safetensors")
-    load_state_dict(unet, safe_load(init_fn))
-
-  sqrt_alphas_cumprod = model.alphas_cumprod.sqrt().realize()
-  sqrt_one_minus_alphas_cumprod = (1 - model.alphas_cumprod).sqrt().realize()
-
-  if len(devices) > 1:
-    to_move = [sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod]
-    if version == "v2-mlperf-train": to_move += get_parameters(unet) + get_parameters(model.cond_stage_model)
-    for p in to_move:
-      p.to_(devices)
-    with Context(BEAM=0):
-      Tensor.realize(*to_move)
-
-  return model, unet, sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod

examples/mlperf/lr_schedulers.py

@@ -1,9 +1,7 @@
-import math
-from tinygrad import dtypes, Tensor
+from tinygrad import Tensor, dtypes
 from tinygrad.nn.optim import Optimizer
 
 from extra.lr_scheduler import LR_Scheduler
-from typing import Callable
 
 # https://github.com/mlcommons/training/blob/e237206991d10449d9675d95606459a3cb6c21ad/image_classification/tensorflow2/lars_util.py
 class PolynomialDecayWithWarmup(LR_Scheduler):
@@ -22,39 +20,3 @@ class PolynomialDecayWithWarmup(LR_Scheduler):
     warmup_lr = (self.epoch_counter * (1.0 / self.warmup)) * self.initial_lr
     x = (1 - (self.epoch_counter - self.warmup) / (self.epochs - self.warmup + 1))
     return (self.epoch_counter <= self.warmup).where(warmup_lr, (self.initial_lr - self.end_lr) * x ** self.power + self.end_lr).cast(self.optimizer.lr.dtype)
-
-class CosineAnnealingLRWithWarmup(LR_Scheduler):
-  def __init__(self, optimizer:Optimizer, base_lr, end_lr, warmup_steps:int, decay_steps:int):
-    assert warmup_steps > 0 and decay_steps > 0
-    super().__init__(optimizer)
-    self.base_lr = base_lr
-    self.end_lr = end_lr
-    self.warmup_steps = warmup_steps
-    self.decay_steps = decay_steps
-    # set lr for first warmup step
-    self.optimizer.lr.assign(self.get_lr()).realize()
-
-  def get_lr(self):
-    warmup_lr = ((self.epoch_counter+1) / self.warmup_steps) * self.base_lr
-    decay_lr = self.end_lr + 0.5 * (self.base_lr-self.end_lr) * (1 + (((self.epoch_counter+1-self.warmup_steps)/self.decay_steps) * math.pi).cos())
-    return (self.epoch_counter < self.warmup_steps).where(warmup_lr, decay_lr).cast(self.optimizer.lr.dtype)
-
-# Reference: https://github.com/mlcommons/training/blob/64b14a9abc74e08779a175abca7d291f8c957632/stable_diffusion/ldm/lr_scheduler.py, Lines 36-97
-class LambdaLinearScheduler:
-  def __init__(self, warm_up_steps:int, f_min:float, f_max:float, f_start:float, cycle_lengths:int):
-    self.lr_warm_up_steps, self.f_min, self.f_max, self.f_start, self.cycle_lengths = warm_up_steps, f_min, f_max, f_start, cycle_lengths
-
-  def schedule(self, n:Tensor) -> Tensor:
-    warm_up = (n < self.lr_warm_up_steps)
-    f_warm_up = (self.f_max - self.f_start) / self.lr_warm_up_steps * n + self.f_start
-    return warm_up.where(f_warm_up, self.f_min + (self.f_max - self.f_min) * (self.cycle_lengths - n) / (self.cycle_lengths))
-
-# based on torch.optim.lr_scheduler.LambdaLR
-class LambdaLR(LR_Scheduler):
-  def __init__(self, optimizer:Optimizer, base_lr:Tensor, lr_lambda:Callable):
-    super().__init__(optimizer)
-    self.base_lr, self.lr_lambda = base_lr, lr_lambda
-    self.step()
-
-  def get_lr(self):
-    return self.base_lr * self.lr_lambda(self.epoch_counter - 1)

examples/mlperf/metrics.py

@@ -1,6 +1,6 @@
-import re, string
+import re
+import string
 from collections import Counter
-from tinygrad import Tensor
 
 def levenshtein(a, b):
   n, m = len(a), len(b)
@@ -59,11 +59,3 @@ def f1_score(x, y):
   p = ns / len(xt)
   r = ns / len(yt)
   return 2 * p * r / (p + r)
-
-def log_perplexity(logit:Tensor, target:Tensor, ignore_index:int|None=None):
-  # logit has shape (n_samples, seq_len, vocab_size), target has shape (n_samples, seq_len)
-  assert logit.ndim == 3, logit.ndim
-  assert target.ndim == 2, target.ndim
-  assert logit.shape[:2] == target.shape, f"{logit.shape[:2]=}, {target.shape=}"
-  log_prob = logit.log_softmax(axis=-1)
-  return log_prob.transpose(1, 2).nll_loss(target, ignore_index=ignore_index)

examples/mlperf/model_eval.py

@@ -1,66 +1,65 @@
-import time, math, os
+import time
 start = time.perf_counter()
 from pathlib import Path
 import numpy as np
 from tinygrad import Tensor, Device, dtypes, GlobalCounters, TinyJit
 from tinygrad.nn.state import get_parameters, load_state_dict, safe_load
-from tinygrad.helpers import getenv, Context, prod
-from extra.bench_log import BenchEvent, WallTimeEvent
+from tinygrad.helpers import getenv
 def tlog(x): print(f"{x:25s}  @ {time.perf_counter()-start:5.2f}s")
 
 def eval_resnet():
-  with WallTimeEvent(BenchEvent.FULL):
-    # Resnet50-v1.5
-    from extra.models.resnet import ResNet50
-    tlog("imports")
-    GPUS = [f'{Device.DEFAULT}:{i}' for i in range(getenv("GPUS", 6))]
-    for x in GPUS: Device[x]
-    tlog("got devices")    # NOTE: this is faster with rocm-smi running
+  Tensor.no_grad = True
+  # Resnet50-v1.5
+  from extra.models.resnet import ResNet50
+  tlog("imports")
+  GPUS = [f'{Device.DEFAULT}:{i}' for i in range(getenv("GPUS", 6))]
+  for x in GPUS: Device[x]
+  tlog("got devices")    # NOTE: this is faster with rocm-smi running
 
-    class ResnetRunner:
-      def __init__(self, device=None):
-        self.mdl = ResNet50()
-        for x in get_parameters(self.mdl) if device else []: x.to_(device)
-        if (fn:=getenv("RESNET_MODEL", "")): load_state_dict(self.mdl, safe_load(fn))
-        else: self.mdl.load_from_pretrained()
-        self.input_mean = Tensor([0.485, 0.456, 0.406], device=device).reshape(1, -1, 1, 1)
-        self.input_std = Tensor([0.229, 0.224, 0.225], device=device).reshape(1, -1, 1, 1)
-      def __call__(self, x:Tensor) -> Tensor:
-        x = x.permute([0,3,1,2]).cast(dtypes.float32) / 255.0
-        x -= self.input_mean
-        x /= self.input_std
-        return self.mdl(x).log_softmax().argmax(axis=1).realize()
+  class ResnetRunner:
+    def __init__(self, device=None):
+      self.mdl = ResNet50()
+      for x in get_parameters(self.mdl) if device else []: x.to_(device)
+      if (fn:=getenv("RESNET_MODEL", "")): load_state_dict(self.mdl, safe_load(fn))
+      else: self.mdl.load_from_pretrained()
+      self.input_mean = Tensor([0.485, 0.456, 0.406], device=device).reshape(1, -1, 1, 1)
+      self.input_std = Tensor([0.229, 0.224, 0.225], device=device).reshape(1, -1, 1, 1)
+    def __call__(self, x:Tensor) -> Tensor:
+      x = x.permute([0,3,1,2]).cast(dtypes.float32) / 255.0
+      x -= self.input_mean
+      x /= self.input_std
+      return self.mdl(x).log_softmax().argmax(axis=1).realize()
 
-    mdl = TinyJit(ResnetRunner(GPUS))
-    tlog("loaded models")
+  mdl = TinyJit(ResnetRunner(GPUS))
+  tlog("loaded models")
 
-    # evaluation on the mlperf classes of the validation set from imagenet
-    from examples.mlperf.dataloader import batch_load_resnet
-    iterator = batch_load_resnet(getenv("BS", 128*6), val=getenv("VAL", 1), shuffle=False, pad_first_batch=True)
-    def data_get():
-      x,y,cookie = next(iterator)
-      return x.shard(GPUS, axis=0).realize(), y, cookie
-    n,d = 0,0
-    proc = data_get()
-    tlog("loaded initial data")
-    st = time.perf_counter()
-    while proc is not None:
-      GlobalCounters.reset()
-      proc = (mdl(proc[0]), proc[1], proc[2])  # this frees the images
-      run = time.perf_counter()
-      # load the next data here
-      try: next_proc = data_get()
-      except StopIteration: next_proc = None
-      nd = time.perf_counter()
-      y = np.array(proc[1])
-      proc = (proc[0].numpy() == y) & (y != -1)  # this realizes the models and frees the cookies
-      n += proc.sum()
-      d += (y != -1).sum()
-      et = time.perf_counter()
-      tlog(f"****** {n:5d}/{d:5d}  {n*100.0/d:.2f}% -- {(run-st)*1000:7.2f} ms to enqueue, {(et-run)*1000:7.2f} ms to realize ({(nd-run)*1000:7.2f} ms fetching). {(len(proc))/(et-st):8.2f} examples/sec. {GlobalCounters.global_ops*1e-12/(et-st):5.2f} TFLOPS")
-      st = et
-      proc, next_proc = next_proc, None
-    tlog("done")
+  # evaluation on the mlperf classes of the validation set from imagenet
+  from examples.mlperf.dataloader import batch_load_resnet
+  iterator = batch_load_resnet(getenv("BS", 128*6), val=getenv("VAL", 1), shuffle=False, pad_first_batch=True)
+  def data_get():
+    x,y,cookie = next(iterator)
+    return x.shard(GPUS, axis=0).realize(), y, cookie
+  n,d = 0,0
+  proc = data_get()
+  tlog("loaded initial data")
+  st = time.perf_counter()
+  while proc is not None:
+    GlobalCounters.reset()
+    proc = (mdl(proc[0]), proc[1], proc[2])  # this frees the images
+    run = time.perf_counter()
+    # load the next data here
+    try: next_proc = data_get()
+    except StopIteration: next_proc = None
+    nd = time.perf_counter()
+    y = np.array(proc[1])
+    proc = (proc[0].numpy() == y) & (y != -1)  # this realizes the models and frees the cookies
+    n += proc.sum()
+    d += (y != -1).sum()
+    et = time.perf_counter()
+    tlog(f"****** {n:5d}/{d:5d}  {n*100.0/d:.2f}% -- {(run-st)*1000:7.2f} ms to enqueue, {(et-run)*1000:7.2f} ms to realize ({(nd-run)*1000:7.2f} ms fetching). {(len(proc))/(et-st):8.2f} examples/sec. {GlobalCounters.global_ops*1e-12/(et-st):5.2f} TFLOPS")
+    st = et
+    proc, next_proc = next_proc, None
+  tlog("done")
 
 def eval_unet3d():
   # UNet3D
@@ -241,305 +240,10 @@ def eval_mrcnn():
   evaluate_predictions_on_coco(bbox_output, iou_type='bbox')
   evaluate_predictions_on_coco(mask_output, iou_type='segm')
 
-def eval_llama3():
-  from extra.models.llama import Transformer
-  from examples.llama3 import MODEL_PARAMS, load, convert_from_huggingface
-  from tinygrad.helpers import tqdm
-
-  BASEDIR = Path(getenv("BASEDIR", "/raid/datasets/c4/"))
-  BS = getenv("BS", 4)
-  SMALL = getenv("SMALL", 0)
-  SEQLEN = getenv("SEQLEN", 8192)
-  MODEL_PATH = Path(getenv("MODEL_PATH", "/raid/weights/llama31_8b/"))
-
-  params = MODEL_PARAMS[getenv("LLAMA3_SIZE", "8B")]["args"]
-  params = params | {"vocab_size": 32000} if not SMALL else params
-  if (llama_layers:=getenv("LLAMA_LAYERS")) != 0: params['n_layers'] = llama_layers
-  model = Transformer(**params, max_context=SEQLEN, jit=False, disable_kv_cache=True)
-
-  # load weights
-  weights = load(str(MODEL_PATH / "model.safetensors.index.json"))
-  if "model.embed_tokens.weight" in weights:
-    print("converting from huggingface format")
-    weights = convert_from_huggingface(weights, params["n_layers"], params["n_heads"], params["n_kv_heads"])
-
-  load_state_dict(model, weights, strict=False, consume=True)
-
-  @TinyJit
-  def eval_step(model, tokens):
-    logits:Tensor = model(tokens[:, :-1], start_pos=0, temperature=math.nan)
-    loss = logits.sparse_categorical_crossentropy(tokens[:, 1:])
-    return loss.flatten().float()
-
-  if SMALL:
-    from examples.mlperf.dataloader import batch_load_llama3_small
-    iter = batch_load_llama3_small(BS, 5760, SEQLEN, BASEDIR, val=True)
-  else:
-    from examples.mlperf.dataloader import batch_load_llama3
-    iter = batch_load_llama3(BS, 5760, SEQLEN, BASEDIR, val=True)
-
-  losses = []
-  for tokens in tqdm(iter, total=5760//BS):
-    GlobalCounters.reset()
-    losses += eval_step(model, tokens).tolist()
-    tqdm.write(f"loss: {np.mean(losses)}")
-
-  log_perplexity = np.mean(losses)
-  print(f"Log Perplexity: {log_perplexity}")
-
-# NOTE: BEAM hangs on 8xmi300x with DECODE_BS=384 in final realize below; function is declared here for external testing
-@TinyJit
-def vae_decode(x:Tensor, vae, disable_beam=False) -> Tensor:
-  from examples.stable_diffusion import AutoencoderKL
-  assert isinstance(vae, AutoencoderKL)
-  x = vae.post_quant_conv(1./0.18215 * x)
-
-  x = vae.decoder.conv_in(x)
-  x = vae.decoder.mid(x)
-  for i, l in enumerate(vae.decoder.up[::-1]):
-    print("decode", x.shape)
-    for b in l['block']: x = b(x)
-    if 'upsample' in l:
-      bs,c,py,px = x.shape
-      x = x.reshape(bs, c, py, 1, px, 1).expand(bs, c, py, 2, px, 2).reshape(bs, c, py*2, px*2)
-      x = l['upsample']['conv'](x)
-    if i == len(vae.decoder.up) - 1 and disable_beam:
-      with Context(BEAM=0): x.realize()
-    else: x.realize()
-  x = vae.decoder.conv_out(vae.decoder.norm_out(x).swish())
-
-  x = ((x + 1.0) / 2.0).clip(0.0, 1.0)
-  return x
-
-def eval_stable_diffusion():
-  import csv, PIL, sys
-  from tqdm import tqdm
-  from examples.mlperf.initializers import init_stable_diffusion, gelu_erf
-  from examples.stable_diffusion import AutoencoderKL
-  from extra.models.unet import UNetModel
-  from tinygrad.nn.state import load_state_dict, torch_load
-  from tinygrad.helpers import BEAM
-  from extra.models import clip
-  from extra.models.clip import FrozenOpenClipEmbedder
-  from extra.models.clip import OpenClipEncoder
-  from extra.models.inception import FidInceptionV3
-
-  config = {}
-  GPUS               = config["GPUS"]                   = [f"{Device.DEFAULT}:{i}" for i in range(getenv("GPUS", 1))]
-  for x in GPUS: Device[x]
-  print(f"running eval on {GPUS}")
-  seed               = config["seed"]                   = getenv("SEED", 12345)
-  CKPTDIR            = config["CKPTDIR"]                = Path(getenv("CKPTDIR", "./checkpoints"))
-  DATADIR            = config["DATADIR"]                = Path(getenv("DATADIR", "./datasets"))
-  CONTEXT_BS         = config["CONTEXT_BS"]             = getenv("CONTEXT_BS", 1 * len(GPUS))
-  DENOISE_BS         = config["DENOISE_BS"]             = getenv("DENOISE_BS", 1 * len(GPUS))
-  DECODE_BS          = config["DECODE_BS"]              = getenv("DECODE_BS", 1 * len(GPUS))
-  INCEPTION_BS       = config["INCEPTION_BS"]           = getenv("INCEPTION_BS", 1 * len(GPUS))
-  CLIP_BS            = config["CLIP_BS"]                = getenv("CLIP_BS", 1 * len(GPUS))
-  EVAL_CKPT_DIR      = config["EVAL_CKPT_DIR"]          = getenv("EVAL_CKPT_DIR", "")
-  STOP_IF_CONVERGED  = config["STOP_IF_CONVERGED"]      = getenv("STOP_IF_CONVERGED", 0)
-
-  if (WANDB := getenv("WANDB", "")):
-    import wandb
-    wandb.init(config=config, project="MLPerf-Stable-Diffusion")
-
-  assert EVAL_CKPT_DIR != "", "provide a directory with checkpoints to be evaluated"
-  print(f"running eval on checkpoints in {EVAL_CKPT_DIR}\nSEED={seed}")
-  eval_queue:list[tuple[int, Path]] = []
-  for p in Path(EVAL_CKPT_DIR).iterdir():
-    if p.name.endswith(".safetensors"):
-      ckpt_iteration = p.name.split(".safetensors")[0]
-      assert ckpt_iteration.isdigit(), f"invalid checkpoint name: {p.name}, expected <digits>.safetensors"
-      eval_queue.append((int(ckpt_iteration), p))
-  assert len(eval_queue), f'no files ending with ".safetensors" were found in {EVAL_CKPT_DIR}'
-  print(sorted(eval_queue, reverse=True))
-
-  Tensor.manual_seed(seed)  # seed for weight initialization
-  model, unet, sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod = init_stable_diffusion("v2-mlperf-eval", CKPTDIR / "sd" / "512-base-ema.ckpt", GPUS)
-
-  # load prompts for generating images for validation; 2 MB of data total
-  with open(DATADIR / "coco2014" / "val2014_30k.tsv") as f:
-    reader = csv.DictReader(f, delimiter="\t")
-    eval_inputs:list[dict] = [{"image_id": int(row["image_id"]), "id": int(row["id"]), "caption": row["caption"]} for row in reader]
-  assert len(eval_inputs) == 30_000
-  # NOTE: the clip weights are the same between model.cond_stage_model and clip_encoder
-  eval_timesteps = list(reversed(range(1, 1000, 20)))
-
-  original_device, Device.DEFAULT = Device.DEFAULT, "CPU"
-  # The choice of alphas_prev[0] = alphas_cumprod[0] seems arbitrary, but it's how the mlperf ref does it:
-  #   alphas_prev = np.asarray([alphacums[0]] + alphacums[ddim_timesteps[:-1]].tolist())
-  eval_alphas_prev = model.alphas_cumprod[0:1].cat(model.alphas_cumprod[list(range(1, 1000, 20))[:-1]]).to(GPUS).realize()
-  inception = FidInceptionV3().load_from_pretrained(CKPTDIR / "inception" / "pt_inception-2015-12-05-6726825d.pth")
-  vision_cfg = {'width': 1280, 'layers': 32, 'd_head': 80, 'image_size': 224, 'patch_size': 14}
-  text_cfg = {'width': 1024, 'n_heads': 16, 'layers': 24, 'vocab_size': 49408, 'ctx_length': 77}
-  clip.gelu = gelu_erf
-  clip_encoder = OpenClipEncoder(1024, text_cfg, vision_cfg)
-  loaded = torch_load(CKPTDIR / "clip" / "open_clip_pytorch_model.bin")
-  loaded.update({"attn_mask": clip_encoder.attn_mask, "mean": clip_encoder.mean, "std": clip_encoder.std})
-  load_state_dict(clip_encoder, loaded)
-  Device.DEFAULT=original_device
-
-  @TinyJit
-  def denoise_step(x:Tensor, x_x:Tensor, t_t:Tensor, uc_c:Tensor, sqrt_alphas_cumprod_t:Tensor, sqrt_one_minus_alphas_cumprod_t:Tensor,
-                    alpha_prev:Tensor, unet:UNetModel, GPUS) -> Tensor:
-    out_uncond, out = unet(x_x, t_t, uc_c).to("CPU").reshape(-1, 2, 4, 64, 64).chunk(2, dim=1)
-    out_uncond = out_uncond.squeeze(1).shard(GPUS,axis=0)
-    out = out.squeeze(1).shard(GPUS,axis=0)
-    v_t = out_uncond + 8.0 * (out - out_uncond)
-    e_t = sqrt_alphas_cumprod_t * v_t + sqrt_one_minus_alphas_cumprod_t * x
-    pred_x0 = sqrt_alphas_cumprod_t * x - sqrt_one_minus_alphas_cumprod_t * v_t
-    dir_xt = (1. - alpha_prev).sqrt() * e_t
-    x_prev = alpha_prev.sqrt() * pred_x0 + dir_xt
-    return x_prev.realize()
-
-  def shard_tensor(t:Tensor) -> Tensor: return t.shard(GPUS, axis=0) if len(GPUS) > 1 else t.to(GPUS[0])
-  def get_batch(whole:Tensor, i:int, bs:int) -> tuple[Tensor, int]:
-    batch = whole[i: i + bs].to("CPU")
-    if (unpadded_bs:=batch.shape[0]) < bs:
-      batch = batch.cat(batch[-1:].expand(bs - unpadded_bs, *batch[-1].shape))
-    return batch, unpadded_bs 
-
-  @Tensor.train(mode=False)
-  def eval_unet(eval_inputs:list[dict], unet:UNetModel, cond_stage:FrozenOpenClipEmbedder, first_stage:AutoencoderKL,
-                inception:FidInceptionV3, clip:OpenClipEncoder) -> tuple[float, float]:
-    # Eval is divided into 5 jits, one per model
-    # It doesn't make sense to merge these jits, e.g. unet repeats 50 times in isolation; images fork to separate inception/clip
-    # We're generating and scoring 30,000 images per eval, and all the data can flow through one jit at a time
-    # To maximize throughput for each jit, we have only one model/jit on the GPU at a time, and pool outputs from each jit off-GPU
-    for model in (unet, first_stage, inception, clip):
-      Tensor.realize(*[p.to_("CPU") for p in get_parameters(model)])
-
-    uc_written = False
-    models = (cond_stage, unet, first_stage, inception, clip)
-    jits = (jit_context:=TinyJit(cond_stage.embed_tokens), denoise_step, vae_decode, jit_inception:=TinyJit(inception),
-            jit_clip:=TinyJit(clip.get_clip_score))
-    all_bs = (CONTEXT_BS, DENOISE_BS, DECODE_BS, INCEPTION_BS, CLIP_BS)
-    if (EVAL_SAMPLES:=getenv("EVAL_SAMPLES", 0)) and EVAL_SAMPLES > 0:
-      eval_inputs = eval_inputs[0:EVAL_SAMPLES]
-    output_shapes = [(ns:=len(eval_inputs),77), (ns,77,1024), (ns,4,64,64), (ns,3,512,512), (ns,2048), (ns,)]
-    # Writing progress to disk lets us resume eval if we crash
-    stages = ["tokens", "embeds", "latents", "imgs", "inception", "clip"]
-    disk_tensor_names, disk_tensor_shapes = stages + ["end", "uc"], output_shapes + [(6,), (1,77,1024)]
-    if not all(os.path.exists(f"{EVAL_CKPT_DIR}/{name}.bytes") for name in disk_tensor_names):
-      for name, shape in zip(disk_tensor_names, disk_tensor_shapes):
-        file = Path(f"{EVAL_CKPT_DIR}/{name}.bytes")
-        file.unlink(missing_ok=True)
-        with file.open("wb") as f: f.truncate(prod(shape) * 4)
-    progress = {name: Tensor.empty(*shape, device=f"disk:{EVAL_CKPT_DIR}/{name}.bytes", dtype=dtypes.int if name in {"tokens", "end"} else dtypes.float)
-                for name, shape in zip(disk_tensor_names, disk_tensor_shapes)}
-
-    def embed_tokens(tokens:Tensor) -> Tensor:
-      nonlocal uc_written
-      if not uc_written:
-        with Context(BEAM=0): progress["uc"].assign(cond_stage.embed_tokens(cond_stage.tokenize("").to(GPUS)).to("CPU").realize()).realize()
-        uc_written = True
-      return jit_context(shard_tensor(tokens))
-
-    def generate_latents(embeds:Tensor) -> Tensor:
-      uc_c = Tensor.stack(progress["uc"].to("CPU").expand(bs, 77, 1024), embeds, dim=1).reshape(-1, 77, 1024)
-      uc_c = shard_tensor(uc_c)
-      x = shard_tensor(Tensor.randn(bs,4,64,64))
-      for step_idx, timestep in enumerate(tqdm(eval_timesteps)):
-        reversed_idx = Tensor([50 - step_idx - 1], device=GPUS)
-        alpha_prev = eval_alphas_prev[reversed_idx]
-        ts = Tensor.full(bs, fill_value=timestep, dtype=dtypes.int, device="CPU")
-        ts_ts = shard_tensor(ts.cat(ts))
-        ts = shard_tensor(ts)
-        sqrt_alphas_cumprod_t = sqrt_alphas_cumprod[ts].reshape(bs, 1, 1, 1)
-        sqrt_one_minus_alphas_cumprod_t = sqrt_one_minus_alphas_cumprod[ts].reshape(bs, 1, 1, 1)
-        x_x = shard_tensor(Tensor.stack(x.to("CPU"), x.to("CPU"), dim=1).reshape(-1, 4, 64, 64))
-        x.assign(denoise_step(x, x_x, ts_ts, uc_c, sqrt_alphas_cumprod_t, sqrt_one_minus_alphas_cumprod_t, alpha_prev, unet, GPUS)).realize()
-      return x
-
-    def decode_latents(latents:Tensor) -> Tensor: return vae_decode(shard_tensor(latents), first_stage, disable_beam=True)
-    def generate_inception(imgs:Tensor) -> Tensor: return jit_inception(shard_tensor(imgs))[:,:,0,0]
-
-    def calc_clip_scores(batch:Tensor, batch_tokens:Tensor) -> Tensor:
-      # Tensor.interpolate does not yet support bicubic, so we use PIL
-      batch = (batch.to(GPUS[0]).permute(0,2,3,1) * 255).clip(0, 255).cast(dtypes.uint8).numpy()
-      batch = [np.array(PIL.Image.fromarray(batch[i]).resize((224,224), PIL.Image.BICUBIC)) for i in range(bs)]
-      batch = shard_tensor(Tensor(np.stack(batch, axis=0).transpose(0,3,1,2), device="CPU").realize())
-      batch = batch.cast(dtypes.float) / 255
-      batch = (batch - model.mean) / model.std
-      batch = jit_clip(shard_tensor(batch_tokens), batch)
-      return batch
-
-    callbacks = (embed_tokens, generate_latents, decode_latents, generate_inception, calc_clip_scores)
-
-    # save every forward pass output to disk; NOTE: this needs ~100 GB disk space because 30k images are large
-    def stage_progress(stage_idx:int) -> int: return progress["end"].to("CPU")[stage_idx].item()
-    if stage_progress(0) < len(eval_inputs):
-      tokens = []
-      for i in tqdm(range(0, len(eval_inputs), CONTEXT_BS)):
-        subset = [cond_stage.tokenize(row["caption"], device="CPU") for row in eval_inputs[i: i+CONTEXT_BS]]
-        tokens.append(Tensor.cat(*subset, dim=0).realize())
-      progress["tokens"].assign(Tensor.cat(*tokens, dim=0).realize()).realize()
-      progress["end"][0:1].assign(Tensor([len(eval_inputs)], dtype=dtypes.int)).realize()
-    prev_stage = "tokens"
-    tokens = progress["tokens"]
-
-    # wrapper code for every model
-    for stage_idx, model, jit, bs, callback in zip(range(1,6), models, jits, all_bs, callbacks):
-      stage = stages[stage_idx]
-      if stage_progress(stage_idx) >= len(eval_inputs):
-        prev_stage = stage
-        continue # use cache
-      t0 = time.perf_counter()
-      print(f"starting eval with model: {model}")
-      if stage_idx == 1: inputs = tokens
-      elif stage_idx == 5: inputs = progress["imgs"]
-      else: inputs = progress[prev_stage]
-
-      Tensor.realize(*[p.to_(GPUS) for p in get_parameters(model)])
-      for batch_idx in tqdm(range(stage_progress(stage_idx), inputs.shape[0], bs)):
-        t1 = time.perf_counter()
-        batch, unpadded_bs = get_batch(inputs, batch_idx, bs)
-        if isinstance(model, OpenClipEncoder): batch = callback(batch, get_batch(tokens, batch_idx, bs)[0].realize())
-        else: batch = callback(batch)
-        # to(GPUS[0]) is necessary for this to work, without that the result is still on GPUS, probably due to a bug
-        batch = batch.to(GPUS[0]).to("CPU")[0:unpadded_bs].realize()
-        progress[stage][batch_idx: batch_idx + bs].assign(batch).realize()
-        # keep track of what our last output was, so we can resume from there if we crash in this loop
-        progress["end"][stage_idx: stage_idx + 1].assign(Tensor([batch_idx + bs], dtype=dtypes.int)).realize()
-        print(f"model: {model}, batch_idx: {batch_idx}, elapsed: {(time.perf_counter() - t1):.2f}")
-      del batch
-        
-      jit.reset()
-      Tensor.realize(*[p.to_("CPU") for p in get_parameters(model)])
-      print(f"done with model: {model}, elapsed: {(time.perf_counter() - t0):.2f}")
-      prev_stage = stage
-
-    inception_stats_fn = str(DATADIR / "coco2014" / "val2014_30k_stats.npz")
-    fid_score = inception.compute_score(progress["inception"].to("CPU"), inception_stats_fn)
-    clip_score = progress["clip"].to(GPUS[0]).mean().item()
-    for name in disk_tensor_names:
-      Path(f"{EVAL_CKPT_DIR}/{name}.bytes").unlink(missing_ok=True)
-    
-    if EVAL_SAMPLES and BEAM:
-      print("BEAM COMPLETE", flush=True) # allows wrapper script to detect BEAM search completion and retry if it failed
-      sys.exit() # Don't eval additional models; we don't care about clip/fid scores when running BEAM on eval sample subset
-
-    return clip_score, fid_score
-
-  # evaluate checkpoints in reverse chronological order
-  for ckpt_iteration, p in sorted(eval_queue, reverse=True):
-    unet_ckpt = safe_load(p)
-    load_state_dict(unet, unet_ckpt)
-    clip_score, fid_score = eval_unet(eval_inputs, unet, model.cond_stage_model, model.first_stage_model, inception, clip_encoder)
-    converged = True if clip_score >= 0.15 and fid_score <= 90 else False
-    print(f"eval results for {EVAL_CKPT_DIR}/{p.name}: clip={clip_score}, fid={fid_score}, converged={converged}")
-    if WANDB:
-      wandb.log({"eval/ckpt_iteration": ckpt_iteration, "eval/clip_score": clip_score, "eval/fid_score": fid_score})
-    if converged and STOP_IF_CONVERGED:
-      print(f"Convergence detected, exiting early before evaluating other checkpoints due to STOP_IF_CONVERGED={STOP_IF_CONVERGED}")
-      sys.exit()
-
-  # for testing
-  return clip_score, fid_score, ckpt_iteration
-
 if __name__ == "__main__":
   # inference only
   Tensor.training = False
+  Tensor.no_grad = True
 
   models = getenv("MODEL", "resnet,retinanet,unet3d,rnnt,bert,mrcnn").split(",")
   for m in models:

examples/mlperf/model_spec.py

@@ -60,6 +60,7 @@ def spec_mrcnn():
 if __name__ == "__main__":
   # inference only for now
   Tensor.training = False
+  Tensor.no_grad = True
 
   for m in getenv("MODEL", "resnet,retinanet,unet3d,rnnt,bert,mrcnn").split(","):
     nm = f"spec_{m}"

examples/mlperf/model_train.py

@@ -1,15 +1,14 @@
-import os, time, math, functools, random, contextlib
+import os, time, math, functools, random
 from pathlib import Path
 import multiprocessing
 
 from tinygrad import Device, GlobalCounters, Tensor, TinyJit, dtypes
 from tinygrad.helpers import getenv, BEAM, WINO, round_up, diskcache_clear, FUSE_CONV_BW, Profiling
-from tinygrad.nn.state import get_parameters, get_state_dict, load_state_dict, safe_load, safe_save
-from tinygrad.nn.optim import LAMB, LARS, SGD, OptimizerGroup, Adam, AdamW
+from tinygrad.nn.state import get_parameters, get_state_dict, safe_load, safe_save
+from tinygrad.nn.optim import LAMB, LARS, SGD, OptimizerGroup, Adam
 
 from extra.lr_scheduler import LRSchedulerGroup
 from examples.mlperf.helpers import get_training_state, load_training_state
-from extra.bench_log import BenchEvent, WallTimeEvent
 # TODO: fix benchmark logging and use tinygrad tqdm
 from tqdm import tqdm
 
@@ -206,25 +205,24 @@ def train_resnet():
     st = time.perf_counter()
     while proc is not None:
       GlobalCounters.reset()
-      with WallTimeEvent(BenchEvent.STEP):
-        (loss, top_1), y, proc = train_step(proc[0], proc[1]), proc[2], proc[3]
+      (loss, top_1), y, proc = train_step(proc[0], proc[1]), proc[2], proc[3]
 
-        pt = time.perf_counter()
+      pt = time.perf_counter()
 
-        if len(prev_cookies) == getenv("STORE_COOKIES", 1): prev_cookies = []  # free previous cookies after gpu work has been enqueued
-        try:
-          if INITMLPERF:
-            next_proc = fake_data_get(BS)
-          else:
-            next_proc = data_get(it)
-        except StopIteration:
-          next_proc = None
+      if len(prev_cookies) == getenv("STORE_COOKIES", 1): prev_cookies = []  # free previous cookies after gpu work has been enqueued
+      try:
+        if INITMLPERF:
+          next_proc = fake_data_get(BS)
+        else:
+          next_proc = data_get(it)
+      except StopIteration:
+        next_proc = None
 
-        dt = time.perf_counter()
+      dt = time.perf_counter()
 
-        device_str = loss.device if isinstance(loss.device, str) else f"{loss.device[0]} * {len(loss.device)}"
-        loss, top_1 = loss.numpy().item(), top_1.numpy().item()
-        top_1_acc = top_1 / sum(yi != -1 for yi in y)
+      device_str = loss.device if isinstance(loss.device, str) else f"{loss.device[0]} * {len(loss.device)}"
+      loss, top_1 = loss.numpy().item(), top_1.numpy().item()
+      top_1_acc = top_1 / sum(yi != -1 for yi in y)
 
       cl = time.perf_counter()
       if BENCHMARK:
@@ -252,10 +250,6 @@ def train_resnet():
         print(f"epoch global_ops: {steps_in_train_epoch * GlobalCounters.global_ops:_}, "
               f"epoch global_mem: {steps_in_train_epoch * GlobalCounters.global_mem:_}")
         # if we are doing beam search, run the first eval too
-        if (assert_time:=getenv("ASSERT_MIN_STEP_TIME")):
-          min_time = min(step_times)
-          assert min_time < assert_time, f"Speed regression, expected min step time of < {assert_time} ms but took: {min_time} ms"
-
         if (TRAIN_BEAM or EVAL_BEAM) and e == start_epoch: break
         return
     if MLLOGGER and RUNMLPERF:
@@ -348,18 +342,17 @@ def train_resnet():
         print(f"saving ckpt to {fn}")
         safe_save(get_training_state(model, optimizer_group, scheduler_group), fn)
 
-
-
 def train_retinanet():
   from contextlib import redirect_stdout
   from examples.mlperf.dataloader import batch_load_retinanet
   from examples.mlperf.initializers import FrozenBatchNorm2dRetinaNet, Conv2dNormalRetinaNet, Conv2dKaimingUniformRetinaNet, Linear, Conv2dRetinaNet
   from extra.datasets.openimages import MLPERF_CLASSES, BASEDIR, download_dataset, normalize, get_dataset_count
-  from extra.models import resnet, retinanet
+  from extra.models import resnet
   from pycocotools.coco import COCO
   from pycocotools.cocoeval import COCOeval
   from tinygrad.helpers import colored
   from typing import Iterator
+  import extra.models.retinanet as retinanet
 
   import numpy as np
 
@@ -374,9 +367,6 @@ def train_retinanet():
   INITMLPERF = getenv("INITMLPERF")
   RUNMLPERF = getenv("RUNMLPERF")
 
-  if INITMLPERF:
-    diskcache_clear()
-
   if getenv("LOGMLPERF"):
     from mlperf_logging import mllog
     import mlperf_logging.mllog.constants as mllog_constants
@@ -395,6 +385,7 @@ def train_retinanet():
 
       MLLOGGER.event(key=mllog_constants.SUBMISSION_BENCHMARK, value=mllog_constants.RETINANET)
 
+      diskcache_clear()
       MLLOGGER.event(key=mllog_constants.CACHE_CLEAR, value=True)
       MLLOGGER.start(key=mllog_constants.INIT_START)
 
@@ -440,7 +431,7 @@ def train_retinanet():
     optim.zero_grad()
 
     losses = model(normalize(x, GPUS), **kwargs)
-    loss = sum(losses.values())
+    loss = sum([l for l in losses.values()])
 
     (loss * loss_scaler).backward()
     for t in optim.params: t.grad = t.grad / loss_scaler
@@ -614,7 +605,7 @@ def train_retinanet():
 
       if getenv("RESET_STEP", 1): _train_step.reset()
 
-      with Tensor.train(mode=False):
+      with Tensor.train(mode=False), Tensor.test():
         if not RUNMLPERF:
           i, proc = 0, _fake_data_get(EVAL_BS, val=(val:=True))
         else:
@@ -797,6 +788,7 @@ def train_unet3d():
     return loss.realize()
 
   @Tensor.train(mode=False)
+  @Tensor.test()
   def eval_step(model, x, y):
     y_hat, y = sliding_window_inference(model, x, y, gpus=GPUS)
     y_hat, y = Tensor(y_hat), Tensor(y, requires_grad=False)
@@ -899,7 +891,7 @@ def train_unet3d():
 
         if mean_dice >= TARGET_METRIC:
           is_successful = True
-          save_checkpoint(get_state_dict(model), "./ckpts/unet3d.safe")
+          save_checkpoint(get_state_dict(model), f"./ckpts/unet3d.safe")
         elif mean_dice < 1e-6:
           print("Model diverging. Aborting.")
           diverged = True
@@ -920,26 +912,18 @@ def train_rnnt():
   pass
 
 @TinyJit
-def train_step_bert(model, optimizer, scheduler, loss_scaler:float, GPUS, grad_acc:int, **kwargs):
+def train_step_bert(model, optimizer, scheduler, loss_scaler:float, input_ids:Tensor, segment_ids:Tensor, attention_mask:Tensor,
+                    masked_positions:Tensor, masked_lm_ids:Tensor, masked_lm_weights:Tensor, next_sentence_labels:Tensor, GPUS):
+  for t in [input_ids, segment_ids, attention_mask, masked_positions, masked_lm_ids, masked_lm_weights, next_sentence_labels]:
+    if len(GPUS) > 1: t.shard_(GPUS, axis=0)
+    else: t.to_(GPUS[0])
   optimizer.zero_grad()
 
-  for i in range(grad_acc):
-    input_ids, segment_ids = kwargs[f"input_ids{i}"], kwargs[f"segment_ids{i}"]
-    # NOTE: these two have different names
-    attention_mask, masked_positions = kwargs[f"input_mask{i}"], kwargs[f"masked_lm_positions{i}"]
-    masked_lm_ids, masked_lm_weights, next_sentence_labels = kwargs[f"masked_lm_ids{i}"], kwargs[f"masked_lm_weights{i}"], kwargs[f"next_sentence_labels{i}"]
+  lm_logits, seq_relationship_logits = model(input_ids, attention_mask, masked_positions, segment_ids)
+  loss = model.loss(lm_logits, seq_relationship_logits, masked_lm_ids, masked_lm_weights, next_sentence_labels)
+  (loss * loss_scaler).backward()
 
-    for t in [input_ids, segment_ids, attention_mask, masked_positions, masked_lm_ids, masked_lm_weights, next_sentence_labels]:
-      if len(GPUS) > 1: t.shard_(GPUS, axis=0)
-      else: t.to_(GPUS[0])
-
-    lm_logits, seq_relationship_logits = model(input_ids, attention_mask, masked_positions, segment_ids)
-    loss = model.loss(lm_logits, seq_relationship_logits, masked_lm_ids, masked_lm_weights, next_sentence_labels)
-    (loss * loss_scaler).backward()
-    # TODO: OOM without this realize with large grad_acc
-    Tensor.realize(*[p.grad for p in optimizer.params])
-
-  global_norm = Tensor(0.0, dtype=dtypes.float32, device=optimizer[0].device)
+  global_norm = Tensor([0.0], dtype=dtypes.float32, device=optimizer[0].device)
   for p in optimizer.params:
     p.grad = p.grad / loss_scaler
     global_norm += p.grad.float().square().sum()
@@ -994,7 +978,7 @@ def train_bert():
     MLLOGGER.logger.propagate = False
 
     if INITMLPERF:
-      assert BENCHMARK, "BENCHMARK must be set for INITMLPERF"
+      assert BENCHMARK, f"BENCHMARK must be set for INITMLPERF"
       MLLOGGER.event(key=mllog_constants.SUBMISSION_ORG, value="tinycorp")
       MLLOGGER.event(key=mllog_constants.SUBMISSION_PLATFORM, value=getenv("SUBMISSION_PLATFORM", "tinybox"))
       MLLOGGER.event(key=mllog_constants.SUBMISSION_DIVISION, value=mllog_constants.CLOSED)
@@ -1013,19 +997,16 @@ def train_bert():
     MLLOGGER = None
 
   # ** hyperparameters **
-  BS                 = config["BS"]                     = getenv("BS", 11 * len(GPUS) if dtypes.default_float in (dtypes.float16, dtypes.bfloat16) else 8 * len(GPUS))
-  grad_acc           = config["GRADIENT_ACC_STEPS"]     = getenv("GRADIENT_ACC_STEPS", 1)
-  # TODO: mlperf logging
-  GBS                = config["GLOBAL_BATCH_SIZE"]      = BS * grad_acc
+  BS                 = config["GLOBAL_BATCH_SIZE"]      = getenv("BS", 11 * len(GPUS) if dtypes.default_float in (dtypes.float16, dtypes.bfloat16) else 8 * len(GPUS))
   EVAL_BS            = config["EVAL_BS"]                = getenv("EVAL_BS", 1 * len(GPUS))
-  max_lr             = config["OPT_BASE_LEARNING_RATE"] = getenv("OPT_BASE_LEARNING_RATE", 0.000175 * math.sqrt(GBS/96))
+  max_lr             = config["OPT_BASE_LEARNING_RATE"] = getenv("OPT_BASE_LEARNING_RATE", 0.000175 * math.sqrt(BS/96))
   opt_lamb_beta_1    = config["OPT_LAMB_BETA_1"]        = getenv("OPT_LAMB_BETA_1", 0.9)
   opt_lamb_beta_2    = config["OPT_LAMB_BETA_2"]        = getenv("OPT_LAMB_BETA_2", 0.999)
 
-  train_steps        = config["TRAIN_STEPS"]            = getenv("TRAIN_STEPS", 3600000 // GBS)
+  train_steps        = config["TRAIN_STEPS"]            = getenv("TRAIN_STEPS", 3600000 // BS)
   warmup_steps       = config["NUM_WARMUP_STEPS"]       = getenv("NUM_WARMUP_STEPS", 1)
   max_eval_steps     = config["MAX_EVAL_STEPS"]         = getenv("MAX_EVAL_STEPS", (10000 + EVAL_BS - 1) // EVAL_BS) # EVAL_BS * MAX_EVAL_STEPS >= 10000
-  eval_step_freq     = config["EVAL_STEP_FREQ"]         = getenv("EVAL_STEP_FREQ", int((math.floor(0.05 * (230.23 * GBS + 3000000) / 25000) * 25000) / GBS)) # Round down
+  eval_step_freq     = config["EVAL_STEP_FREQ"]         = getenv("EVAL_STEP_FREQ", int((math.floor(0.05 * (230.23 * BS + 3000000) / 25000) * 25000) / BS)) # Round down
   save_ckpt_freq     = config["SAVE_CKPT_FREQ"]         = getenv("SAVE_CKPT_FREQ", 1000)
   keep_ckpt_amount   = config["KEEP_CKPT_AMOUNT"]       = getenv("KEEP_CKPT_AMOUNT", 5)
   save_ckpt_dir      = config["SAVE_CKPT_DIR"]          = getenv("SAVE_CKPT_DIR", "./ckpts")
@@ -1083,7 +1064,7 @@ def train_bert():
   scheduler_wd = PolynomialDecayWithWarmup(optimizer_wd, max_lr, 0, train_steps, warmup_steps, power=poly_power)
   scheduler_no_wd = PolynomialDecayWithWarmup(optimizer_no_wd, max_lr, 0, train_steps, warmup_steps, power=poly_power)
   scheduler_group = LRSchedulerGroup(scheduler_wd, scheduler_no_wd)
-  print(f"training with global batch size {GBS} for one epoch with {train_steps} steps")
+  print(f"training with batch size {BS} for one epoch with {train_steps} steps")
 
   # log mlperf hparams
   if MLLOGGER:
@@ -1132,11 +1113,11 @@ def train_bert():
   # ** train loop **
   wc_start = time.perf_counter()
 
-  i, train_data = start_step, [next(train_it) for _ in range(grad_acc)]
+  i, train_data = start_step, next(train_it)
 
   if RUNMLPERF:
     if MLLOGGER:
-      MLLOGGER.start(key=mllog_constants.EPOCH_START, value=i*GBS, metadata={"epoch_num": i*GBS})
+      MLLOGGER.start(key=mllog_constants.EPOCH_START, value=i*BS, metadata={"epoch_num": i*BS})
 
   while train_data is not None and i < train_steps and not achieved:
     if getenv("TRAIN", 1):
@@ -1144,23 +1125,23 @@ def train_bert():
       BEAM.value = TRAIN_BEAM
       st = time.perf_counter()
       GlobalCounters.reset()
-      with WallTimeEvent(BenchEvent.STEP):
-        data = {f"{k}{i}":v for i,d in enumerate(train_data) for k,v in d.items()}
-        loss, global_norm, lr = train_step_bert(model, optimizer_group, scheduler_group, loss_scaler, GPUS, grad_acc, **data)
+      loss, global_norm, lr = train_step_bert(model, optimizer_group, scheduler_group, loss_scaler,
+        train_data["input_ids"], train_data["segment_ids"], train_data["input_mask"], train_data["masked_lm_positions"], \
+        train_data["masked_lm_ids"], train_data["masked_lm_weights"], train_data["next_sentence_labels"], GPUS)
 
-        pt = time.perf_counter()
+      pt = time.perf_counter()
 
-        try:
-          next_data = [next(train_it) for _ in range(grad_acc)]
-        except StopIteration:
-          next_data = None
+      try:
+        next_data = next(train_it)
+      except StopIteration:
+        next_data = None
 
-        dt = time.perf_counter()
+      dt = time.perf_counter()
 
-        device_str = parameters[0].device if isinstance(parameters[0].device, str) else f"{parameters[0].device[0]} * {len(parameters[0].device)}"
-        loss = loss.item()
-        assert not math.isnan(loss)
-        lr = lr.item()
+      device_str = parameters[0].device if isinstance(parameters[0].device, str) else f"{parameters[0].device[0]} * {len(parameters[0].device)}"
+      loss = loss.item()
+      assert not math.isnan(loss)
+      lr = lr.item()
 
       cl = time.perf_counter()
       if BENCHMARK: step_times.append(cl - st)
@@ -1172,7 +1153,7 @@ def train_bert():
       if WANDB:
         wandb.log({"lr": lr, "train/loss": loss, "train/global_norm": global_norm.item(), "train/step_time": cl - st,
                     "train/python_time": pt - st, "train/data_time": dt - pt, "train/cl_time": cl - dt,
-                    "train/GFLOPS": GlobalCounters.global_ops * 1e-9 / (cl - st), "epoch": (i+1)*GBS})
+                    "train/GFLOPS": GlobalCounters.global_ops * 1e-9 / (cl - st), "epoch": (i+1)*BS})
 
       train_data, next_data = next_data, None
       i += 1
@@ -1187,7 +1168,7 @@ def train_bert():
     # ** eval loop **
     if i % eval_step_freq == 0 or (BENCHMARK and i == BENCHMARK) or i == train_steps:
       if MLLOGGER and RUNMLPERF:
-        MLLOGGER.start(key=mllog_constants.EVAL_START, value=None, metadata={"epoch_num": i*GBS, "step_num": i})
+        MLLOGGER.start(key=mllog_constants.EVAL_START, value=None, metadata={"epoch_num": i*BS, "step_num": i})
       if getenv("RESET_STEP"): train_step_bert.reset()
       elif getenv("FREE_INTERMEDIATE", 1) and train_step_bert.captured is not None: train_step_bert.captured.free_intermediates()
       eval_lm_losses = []
@@ -1237,11 +1218,11 @@ def train_bert():
 
       if WANDB:
         wandb.log({"eval/lm_loss": avg_lm_loss, "eval/clsf_loss": avg_clsf_loss, "eval/lm_accuracy": avg_lm_acc, \
-                    "eval/clsf_accuracy": avg_clsf_acc, "eval/forward_time": avg_fw_time, "epoch": (i+1)*GBS})
+                    "eval/clsf_accuracy": avg_clsf_acc, "eval/forward_time": avg_fw_time, "epoch": (i+1)*BS})
 
       if MLLOGGER and RUNMLPERF:
-        MLLOGGER.end(key=mllog_constants.EVAL_STOP, value=i*GBS, metadata={"epoch_count": i*GBS, "step_num": i, "samples_count": config["EVAL_BS"] * config["MAX_EVAL_STEPS"]})
-        MLLOGGER.event(key=mllog_constants.EVAL_ACCURACY, value=avg_lm_acc, metadata={"epoch_num": i*GBS, "masked_lm_accuracy": avg_lm_acc})
+        MLLOGGER.end(key=mllog_constants.EVAL_STOP, value=i*BS, metadata={"epoch_count": i*BS, "step_num": i, "samples_count": config["EVAL_BS"] * config["MAX_EVAL_STEPS"]})
+        MLLOGGER.event(key=mllog_constants.EVAL_ACCURACY, value=avg_lm_acc, metadata={"epoch_num": i*BS, "masked_lm_accuracy": avg_lm_acc})
 
       # save model if achieved target
       if not achieved and avg_lm_acc >= target:
@@ -1256,10 +1237,10 @@ def train_bert():
         hours = int(total_seconds // 3600)
         minutes = int((total_seconds % 3600) // 60)
         seconds = total_seconds % 60
-        print(f"Reference Convergence point reached after {i * GBS} datasamples and {hours}h{minutes}m{seconds:.2f}s.")
+        print(f"Reference Convergence point reached after {i * BS} datasamples and {hours}h{minutes}m{seconds:.2f}s.")
         achieved = True
         if MLLOGGER and RUNMLPERF:
-          MLLOGGER.event(key=mllog_constants.EPOCH_STOP, value=i*GBS, metadata={"epoch_num": i*GBS})
+          MLLOGGER.event(key=mllog_constants.EPOCH_STOP, value=i*BS, metadata={"epoch_num": i*BS})
           MLLOGGER.end(key=mllog_constants.RUN_STOP, metadata=dict(status=mllog_constants.SUCCESS))
         # stop once hitting the target
         break
@@ -1271,7 +1252,7 @@ def train_bert():
       if MLLOGGER and RUNMLPERF:
         if previous_step:
           MLLOGGER.end(key=mllog_constants.BLOCK_STOP, value=None, metadata={"first_epoch_num": 1, "epoch_num": 1, "first_step_num": i, "step_num": i, "step_count": i - previous_step})
-        MLLOGGER.start(key="checkpoint_start", value=None, metadata={"step_num": i})
+        MLLOGGER.start(key="checkpoint_start", value=None, metadata={"step_num" : i})
       if not os.path.exists(ckpt_dir := save_ckpt_dir): os.mkdir(ckpt_dir)
       if WANDB and wandb.run is not None:
         fn = f"{ckpt_dir}/{time.strftime('%Y%m%d_%H%M%S')}_{wandb.run.id}.safe"
@@ -1287,361 +1268,18 @@ def train_bert():
         os.remove(os.path.join(ckpt_dir, last))
       if MLLOGGER and RUNMLPERF:
         MLLOGGER.end(key="checkpoint_stop", value=None, metadata={"step_num": i})
-        MLLOGGER.start(key=mllog_constants.BLOCK_START, value=None, metadata={"first_epoch_num": 1, "epoch_num": 1, "epoch_count": 1, "samples_count": i * GBS, "step_num": i, "first_step_num": i+1})
+        MLLOGGER.start(key=mllog_constants.BLOCK_START, value=None, metadata={"first_epoch_num": 1, "epoch_num": 1, "epoch_count": 1, "samples_count": i * BS, "step_num": i, "first_step_num": i+1})
         previous_step = i
 
-def train_llama3():
-  from extra.models.llama import Transformer
-  from examples.llama3 import MODEL_PARAMS
-  from examples.mlperf.lr_schedulers import CosineAnnealingLRWithWarmup
-
-  config = {}
-  BASEDIR            = config["BASEDIR"]                = Path(getenv("BASEDIR", "/raid/datasets/c4/"))
-  BS                 = config["BS"]                     = getenv("BS", 16)
-  grad_acc           = config["GRADIENT_ACC_STEPS"]     = getenv("GRADIENT_ACC_STEPS", 1)
-  GBS                = config["GLOBAL_BATCH_SIZE"]      = BS * grad_acc
-  SEED               = config["SEED"]                   = getenv("SEED", 5760)
-  SEQLEN             = config["SEQLEN"]                 = getenv("SEQLEN", 8192)
-  TRAIN_ON_VAL       = config["TRAIN_ON_VAL"]           = getenv("TRAIN_ON_VAL", 0)
-  SMALL              = config["SMALL"]                  = getenv("SMALL", 0)
-  SAMPLES            = config["SAMPLES"]                = getenv("SAMPLES", 5_760 if TRAIN_ON_VAL else 1_200_000 * 1152)
-  EVAL_FREQ          = config["EVAL_FREQ"]              = getenv("EVAL_FREQ", 46080)
-  EVAL_BS            = config["EVAL_BS"]                = getenv("EVAL_BS", 16)
-  EVAL_TARGET        = config["EVAL_TARGET"]            = getenv("EVAL_TARGET", 5.6)
-
-  # LR=1e-4 TRAIN_ON_VAL=1 DEFAULT_FLOAT=bfloat16 FUSE_ARANGE=1 JITBEAM=2 OPTIM_DTYPE=bfloat16 LLAMA3_SIZE=1B WARMUP_STEPS=36 DECAY_STEPS=360 SEQLEN=512 PYTHONPATH=. AMD=1 AMD_LLVM=0 MODEL=llama3 python3 examples/mlperf/model_train.py
-  # trains to 7
-
-  opt_adamw_beta_1 = 0.9
-  opt_adamw_beta_2 = 0.95
-  opt_adamw_epsilon = 1e-5
-  opt_adamw_weight_decay = 0.1
-
-  opt_gradient_clip_norm = 1.0
-  opt_learning_rate_warmup_steps = getenv("WARMUP_STEPS", math.ceil(8000 * 1152 / GBS))
-  opt_learning_rate_decay_steps = getenv("MAX_STEPS", math.ceil(1_200_000 * 1152 / GBS)) - opt_learning_rate_warmup_steps
-  opt_base_learning_rate = getenv("LR", 8e-5 * GBS / 1152)  # NOTE: cannot change for benchmark
-  opt_end_learning_rate = getenv("END_LR", 8e-7)
-
-  # TODO: confirm weights are in bf16
-  # vocab_size from the mixtral tokenizer
-  params = MODEL_PARAMS[getenv("LLAMA3_SIZE", "8B")]["args"]
-  params = params | {"vocab_size": 32000} if not SMALL else params
-  if (llama_layers:=getenv("LLAMA_LAYERS")) != 0: params['n_layers'] = llama_layers
-  model = Transformer(**params, max_context=SEQLEN, jit=False, disable_kv_cache=True)
-
-  if getenv("FAKEDATA"):
-    for v in get_parameters(model):
-      v = v.assign(Tensor.empty(v.shape))
-
-  if (DP := getenv("DP", 1)) > 1:
-    device = tuple(f"{Device.DEFAULT}:{i}" for i in range(DP))
-    for v in get_parameters(model):
-      v.shard_(device, axis=None)
-
-  if (MP := getenv("MP", 1)) > 1:
-    device = tuple(f"{Device.DEFAULT}:{i}" for i in range(MP))
-    for k,v in get_state_dict(model).items():
-      if 'scale' in k: v.shard_(device, axis=None)  # from quantized
-      elif '.attention.wq' in k: v.shard_(device, axis=0)
-      elif '.attention.wk' in k: v.shard_(device, axis=0)
-      elif '.attention.wv' in k: v.shard_(device, axis=0)
-      elif '.attention.wo' in k: v.shard_(device, axis=1)
-      elif '.feed_forward.w1.' in k: v.shard_(device, axis=0)
-      elif '.feed_forward.w2.' in k: v.shard_(device, axis=1)
-      elif '.feed_forward.w3.' in k: v.shard_(device, axis=0)
-      elif 'tok_embeddings.weight' in k: v.shard_(device, axis=0)
-      elif 'output.weight' in k: v.shard_(device, axis=0)
-      else:
-        # attention_norm, ffn_norm, norm
-        v.shard_(device, axis=None)
-      # prevents memory spike on device 0
-      v.realize()
-
-  optim = AdamW(get_parameters(model), lr=0.0,
-                b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay)
-  scheduler = CosineAnnealingLRWithWarmup(optim, opt_base_learning_rate, opt_end_learning_rate, opt_learning_rate_warmup_steps, opt_learning_rate_decay_steps)
-
-  if resume_ckpt := getenv("RESUME_CKPT"):
-    fn = f"./ckpts/llama3_{resume_ckpt}.safe"
-    print(f"loading initial checkpoint from {fn}")
-    load_state_dict(model, safe_load(fn), realize=False)
-
-    fn = f"./ckpts/llama3_{resume_ckpt}_optim.safe"
-    print(f"loading optim checkpoint from {fn}")
-    load_state_dict(scheduler, safe_load(fn), realize=False)
-
-  @TinyJit
-  @Tensor.train()
-  def train_step(model, tokens:Tensor, grad_acc:int):
-    optim.zero_grad()
-    # grad acc
-    for batch in tokens.split(tokens.shape[0]//grad_acc):
-      if (DP := getenv("DP", 1)) > 1:
-        device = tuple(f"{Device.DEFAULT}:{i}" for i in range(DP))
-        batch = batch.shard(device, 0)
-      if (MP := getenv("MP", 1)) > 1:
-        device = tuple(f"{Device.DEFAULT}:{i}" for i in range(MP))
-        batch = batch.shard(device)
-      logits:Tensor = model(batch[:, :-1], start_pos=0, temperature=math.nan)
-      loss = logits.sparse_categorical_crossentropy(batch[:, 1:])
-      loss.backward()
-      Tensor.realize(*[p.grad for p in optim.params])
-    # L2 norm grad clip
-    # https://github.com/NVIDIA/NeMo/blob/3368c3fc0b4a186ab33a1d68a504315100c0b2a6/nemo/collections/nlp/modules/common/megatron/clip_grads.py#L57
-    # https://docs.pytorch.org/docs/stable/generated/torch.nn.utils.clip_grad_norm_.html
-    if not getenv("DISABLE_GRAD_CLIP_NORM"):
-      total_norm = Tensor(0.0, dtype=dtypes.float32, device=optim.params[0].device)
-      for p in optim.params:
-        total_norm += p.grad.float().square().sum()
-      total_norm = total_norm.sqrt().contiguous()
-      for p in optim.params:
-        p.grad = p.grad * (opt_gradient_clip_norm / (total_norm + 1e-6)).clamp(max_=1.0)
-
-    optim.step()
-    scheduler.step()
-
-    lr = optim.lr
-    loss.realize(lr)
-    return loss, lr
-
-  @TinyJit
-  @Tensor.train(False)
-  def eval_step(model, tokens:Tensor):
-    if (DP := getenv("DP", 1)) > 1:
-      device = tuple(f"{Device.DEFAULT}:{i}" for i in range(DP))
-      tokens = tokens.shard(device, 0)
-    if (MP := getenv("MP", 1)) > 1:
-      device = tuple(f"{Device.DEFAULT}:{i}" for i in range(MP))
-      tokens = tokens.shard(device)
-    logits:Tensor = model(tokens[:, :-1], start_pos=0, temperature=math.nan)
-    loss = logits.sparse_categorical_crossentropy(tokens[:, 1:])
-    return loss.flatten().float()
-
-  # ** data iters **
-  def fake_data(bs, samples):
-    for _ in range(samples // bs):
-      yield Tensor.randint(bs, SEQLEN + 1, low=0, high=params["vocab_size"], dtype=dtypes.int32, device=Device.DEFAULT)
-
-  def get_train_iter():
-    if getenv("FAKEDATA", 0):
-      return fake_data(GBS, SAMPLES)
-    else:
-      if SMALL:
-        from examples.mlperf.dataloader import batch_load_llama3_small
-        return batch_load_llama3_small(GBS, SAMPLES, SEQLEN, BASEDIR, seed=SEED, val=bool(TRAIN_ON_VAL))
-      else:
-        from examples.mlperf.dataloader import batch_load_llama3
-        return batch_load_llama3(GBS, SAMPLES, SEQLEN, BASEDIR, seed=SEED, val=bool(TRAIN_ON_VAL))
-
-  def get_eval_iter():
-    if getenv("FAKEDATA", 0):
-      return fake_data(EVAL_BS, 5760)
-    else:
-      if SMALL:
-        from examples.mlperf.dataloader import batch_load_llama3_small
-        return batch_load_llama3_small(EVAL_BS, 5760, SEQLEN, BASEDIR, val=True)
-      else:
-        from examples.mlperf.dataloader import batch_load_llama3
-        return batch_load_llama3(EVAL_BS, 5760, SEQLEN, BASEDIR, val=True)
-
-  iter = get_train_iter()
-  i, sequences_seen = resume_ckpt, 0
-  for tokens in tqdm(iter, total=SAMPLES//GBS):
-    t = time.perf_counter()
-    GlobalCounters.reset()
-    loss, lr = train_step(model, tokens, grad_acc)
-    loss = loss.float().item()
-
-    i += 1
-    sequences_seen += tokens.shape[0]
-
-    tqdm.write(f"{loss:.4f} loss, {lr.item():.12f} LR, {GlobalCounters.mem_used / 1e9:.2f} GB used, {time.perf_counter()-t:.2f} s")
-    if (fname:=getenv("LOSS_FILE", "")):
-      with open(fname, "a") as f:
-        f.write(f"{i} {loss:.4f} {lr.item():.12f} {GlobalCounters.mem_used / 1e9:.2f}\n")
-
-    if (ckpt_freq := getenv("CKPT")) and (i % ckpt_freq == 0 and (i != 1 or ckpt_freq == 1)):
-      tqdm.write("saving checkpoint")
-      if not os.path.exists(ckpt_dir := "./ckpts"): os.mkdir(ckpt_dir)
-      fn = f"{ckpt_dir}/llama3_{i}.safe"
-      safe_save(get_state_dict(model), fn)
-
-      tqdm.write("saving optim checkpoint")
-      fn = f"{ckpt_dir}/llama3_{i}_optim.safe"
-      safe_save(get_state_dict(scheduler), fn)
-
-    if sequences_seen % EVAL_FREQ == 0 and (i != 1 or EVAL_FREQ == 1):
-      tqdm.write(f"evaluating after {sequences_seen} sequences")
-
-      # run eval
-      eval_losses = []
-      eval_iter = get_eval_iter()
-      tqdm.write(f"evaluating {5760//EVAL_BS} batches of {EVAL_BS} sequences")
-
-      for tokens in tqdm(eval_iter, total=5760//EVAL_BS):
-        eval_losses += eval_step(model, tokens).tolist()
-      log_perplexity = Tensor(eval_losses).mean().float().item()
-
-      tqdm.write(f"eval log perplexity: {log_perplexity:.4f}")
-
-      if log_perplexity < EVAL_TARGET:
-        tqdm.write(f"target achieved after {sequences_seen} sequences")
-        if getenv("CKPT"):
-          if not os.path.exists(ckpt_dir := "./ckpts"): os.mkdir(ckpt_dir)
-          fn = f"{ckpt_dir}/llama3.safe"
-          safe_save(get_state_dict(model), fn)
-        break
-
-def train_stable_diffusion():
-  from extra.models.unet import UNetModel
-  from examples.mlperf.dataloader import batch_load_train_stable_diffusion
-  from examples.mlperf.lr_schedulers import LambdaLR, LambdaLinearScheduler
-  from examples.mlperf.initializers import init_stable_diffusion
-  from examples.mlperf.helpers import get_training_state
-  import numpy as np
-
-  config = {}
-  GPUS               = config["GPUS"]                   = [f"{Device.DEFAULT}:{i}" for i in range(getenv("GPUS", 1))]
-  seed               = config["seed"]                   = getenv("SEED", 12345)
-  # ** hyperparameters **
-  BS                 = config["BS"]                     = getenv("BS", 1 * len(GPUS))
-  BASE_LR            = config["LEARNING_RATE"]          = getenv("LEARNING_RATE", 2.5e-7)
-  # https://github.com/mlcommons/training_policies/blob/cfa99da479b8d5931f7a3c67612d021dfb47510a/training_rules.adoc#benchmark_specific_rules
-  # "Checkpoint must be collected every 512,000 images. CEIL(512000 / global_batch_size) if 512000 is not divisible by GBS."
-  # NOTE: It's inferred that "steps" is the unit for the output of the CEIL formula, based on all other cases of CEIL in the rules
-  CKPT_STEP_INTERVAL = config["CKPT_STEP_INTERVAL"]     = getenv("CKPT_STEP_INTERVAL", math.ceil(512_000 / BS))
-  CKPTDIR            = config["CKPTDIR"]                = Path(getenv("CKPTDIR", "./checkpoints"))
-  DATADIR            = config["DATADIR"]                = Path(getenv("DATADIR", "./datasets"))
-  UNET_CKPTDIR       = config["UNET_CKPTDIR"]           = Path(getenv("UNET_CKPTDIR", "./checkpoints"))
-  TOTAL_CKPTS        = config["TOTAL_CKPTS"]            = getenv("TOTAL_CKPTS", 0)
-
-  print(f"training on {GPUS}")
-  lr = BS * BASE_LR
-  print(f"BS={BS}, BASE_LR={BASE_LR}, lr={lr}")
-  print(f"CKPT_STEP_INTERVAL = {CKPT_STEP_INTERVAL}")
-  for x in GPUS: Device[x]
-  if (WANDB := getenv("WANDB", "")):
-    import wandb
-    wandb.init(config=config, project="MLPerf-Stable-Diffusion")
-
-  Tensor.manual_seed(seed)  # seed for weight initialization
-  model, unet, sqrt_alphas_cumprod, sqrt_one_minus_alphas_cumprod = init_stable_diffusion("v2-mlperf-train", CKPTDIR / "sd" / "512-base-ema.ckpt", GPUS)
-
-  optimizer = AdamW(get_parameters(unet))
-  lambda_lr_callback = LambdaLinearScheduler(1000, 1.0, 1.0, 1e-06, 10000000000000).schedule
-  lr_scheduler = LambdaLR(optimizer, Tensor(lr, dtype=dtypes.float, device=optimizer.device), lambda_lr_callback)
-
-  @TinyJit
-  def train_step(mean:Tensor, logvar:Tensor, tokens:Tensor, unet:UNetModel, optimizer:LAMB, lr_scheduler:LambdaLR) -> Tensor:
-    optimizer.zero_grad()
-
-    timestep = Tensor.randint(BS, low=0, high=model.alphas_cumprod.shape[0], dtype=dtypes.int, device=GPUS[0])
-    latent_randn = Tensor.randn(*mean.shape, device=GPUS[0])
-    noise = Tensor.randn(*mean.shape, device=GPUS[0])
-    for t in (mean, logvar, tokens, timestep, latent_randn, noise):
-      t.shard_(GPUS, axis=0)
-
-    std = Tensor.exp(0.5 * logvar.clamp(-30.0, 20.0))
-    latent = (mean + std * latent_randn) * 0.18215
-
-    sqrt_alphas_cumprod_t = sqrt_alphas_cumprod[timestep].reshape(timestep.shape[0], 1, 1, 1)
-    sqrt_one_minus_alphas_cumprod_t = sqrt_one_minus_alphas_cumprod[timestep].reshape(timestep.shape[0], 1, 1, 1)
-    latent_with_noise = sqrt_alphas_cumprod_t * latent + sqrt_one_minus_alphas_cumprod_t * noise
-    v_true = sqrt_alphas_cumprod_t * noise - sqrt_one_minus_alphas_cumprod_t * latent
-
-    context = model.cond_stage_model.embed_tokens(tokens)
-
-    out = unet(latent_with_noise, timestep, context)
-    loss = ((out - v_true) ** 2).mean()
-    del mean, logvar, std, latent, noise, sqrt_alphas_cumprod_t, sqrt_one_minus_alphas_cumprod_t
-    del out, v_true, context, latent_randn, tokens, timestep
-    loss.backward()
-
-    optimizer.step()
-    lr_scheduler.step()
-    loss, out_lr = loss.detach().to("CPU"), optimizer.lr.to("CPU")
-    Tensor.realize(loss, out_lr)
-    return loss, out_lr
-    
-  # checkpointing takes ~9 minutes without this, and ~1 minute with this
-  @TinyJit
-  def ckpt_to_cpu():
-    ckpt = get_training_state(unet, optimizer, lr_scheduler)
-    # move to CPU first so more GPU bufs aren't created (can trigger OOM)
-    for k,v in ckpt.items(): ckpt[k] = v.detach().to("CPU")
-    Tensor.realize(*[v for v in ckpt.values()])
-    for k,v in ckpt.items(): ckpt[k] = v.cast(v.dtype.base).contiguous()
-    Tensor.realize(*[v for v in ckpt.values()])
-    return ckpt
-
-  # training loop
-  dl = batch_load_train_stable_diffusion(f'{DATADIR}/laion-400m/webdataset-moments-filtered/{{00000..00831}}.tar', BS)
-  # for tests
-  saved_checkpoints = []
-
-  train_start_time = time.perf_counter()
-  t0 = t6 = time.perf_counter()
-  for i, batch in enumerate(dl, start=1):
-    loop_time = time.perf_counter() - t0
-    t0 = time.perf_counter()
-    dl_time = t0 - t6
-    GlobalCounters.reset()
-
-    mean, logvar = np.split(np.concatenate(batch["npy"], axis=0), 2, axis=1)
-    mean, logvar = Tensor(mean, dtype=dtypes.float32, device="CPU"), Tensor(logvar, dtype=dtypes.float32, device="CPU")
-    tokens = []
-    for text in batch['txt']: tokens += model.cond_stage_model.tokenizer.encode(text, pad_with_zeros=True)
-    tokens = Tensor(tokens, dtype=dtypes.int32, device="CPU").reshape(-1, 77)
-
-    t1 = time.perf_counter()
-    loss, lr = train_step(mean, logvar, tokens, unet, optimizer, lr_scheduler)
-    loss_item, lr_item = loss.item(), lr.item()
-    t2 = time.perf_counter()
-
-    if i == 3:
-      for _ in range(3): ckpt_to_cpu() # do this at the beginning of run to prevent OOM surprises when checkpointing
-      print("BEAM COMPLETE", flush=True) # allows wrapper script to detect BEAM search completion and retry if it failed
-      
-    total_train_time = time.perf_counter() - train_start_time
-    if WANDB:
-      wandb.log({"train/loss": loss_item, "train/lr": lr_item, "train/loop_time_prev": loop_time, "train/dl_time": dl_time, "train/step": i,
-                    "train/GFLOPS": GlobalCounters.global_ops * 1e-9 / (t2-t1), "train/input_prep_time": t1-t0,
-                    "train/train_step_time": t2-t1, "train/total_time": total_train_time})
-
-      if i == 1 and wandb.run is not None:
-        with open(f"{UNET_CKPTDIR}/wandb_run_id_{wandb.run.id}", "w") as f:
-          f.write(f"wandb.run.id = {wandb.run.id}")
-
-    if i % CKPT_STEP_INTERVAL == 0:
-      # https://github.com/mlcommons/training_policies/blob/cfa99da479b8d5931f7a3c67612d021dfb47510a/training_rules.adoc#benchmark_specific_rules
-      # "evaluation is done offline, the time is not counted towards the submission time."
-      fn = f"{UNET_CKPTDIR}/{i}.safetensors"
-      print(f"saving unet checkpoint at {fn}")
-      saved_checkpoints.append(fn)
-      safe_save({k.replace("model.", ""):v for k,v in ckpt_to_cpu().items() if k.startswith("model.")}, fn)
-      if TOTAL_CKPTS and i == TOTAL_CKPTS * CKPT_STEP_INTERVAL:
-        print(f"ending run after {i} steps ({TOTAL_CKPTS} checkpoints collected)")
-        return saved_checkpoints
-
-    t3 = time.perf_counter()
-    print(f"""step {i}: {GlobalCounters.global_ops * 1e-9 / (t2-t1):9.2f} GFLOPS, mem_used: {GlobalCounters.mem_used / 1e9:.2f} GB,
-  loop_time_prev: {loop_time:.2f}, dl_time: {dl_time:.2f}, input_prep_time: {t1-t0:.2f}, train_step_time: {t2-t1:.2f},
-  t3-t2: {t3-t2:.4f}, loss:{loss_item:.5f}, lr:{lr_item:.3e}, total_train_time:{total_train_time:.2f}
-  """)
-    t6 = time.perf_counter()
+def train_maskrcnn():
+  # TODO: Mask RCNN
+  pass
 
 if __name__ == "__main__":
   multiprocessing.set_start_method('spawn')
-
-  if getenv("INITMLPERF"): bench_log_manager = WallTimeEvent(BenchEvent.MLPERF_INIT)
-  elif getenv("RUNMLPERF"): bench_log_manager = WallTimeEvent(BenchEvent.MLPERF_RUN)
-  else: bench_log_manager = contextlib.nullcontext()
-
   with Tensor.train():
-    for m in getenv("MODEL", "resnet,retinanet,unet3d,rnnt,bert,maskrcnn,stable_diffusion").split(","):
+    for m in getenv("MODEL", "resnet,retinanet,unet3d,rnnt,bert,maskrcnn").split(","):
       nm = f"train_{m}"
       if nm in globals():
         print(f"training {m}")
-        with bench_log_manager:
-          with Profiling(enabled=getenv("PYPROFILE")): globals()[nm]()
+        with Profiling(enabled=getenv("PYPROFILE")): globals()[nm]()

examples/mlperf/scripts/stable_diffusion_downloads.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-# adapted from https://github.com/mlcommons/training/blob/4bdf5c8ed218ad76565a2ba1ac27c919ccc6d689/stable_diffusion/README.md
-
-# setup dirs
-
-DATA=/raid/datasets/stable_diffusion
-
-LAION=$DATA/laion-400m/webdataset-moments-filtered 
-COCO=$DATA/coco2014
-mkdir -p $LAION $COCO
-
-CKPT=/raid/weights/stable_diffusion
-mkdir -p $CKPT/clip $CKPT/sd $CKPT/inception
-
-# download data
-
-# if rclone isn't installed system-wide / in your PATH, put the executable path in quotes below
-#RCLONE=""
-RCLONE="rclone"
-
-## VAE-encoded image latents, from 6.1M image subset of laion-400m
-## about 1 TB for whole download
-$RCLONE config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
-$RCLONE copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/moments-webdataset-filtered/ ${LAION} --include="*.tar" -P
-$RCLONE copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/moments-webdataset-filtered/sha512sums.txt ${LAION} -P
-cd $LAION && grep -E '\.tar$' sha512sums.txt | sha512sum -c --quiet - && \
-  echo "All .tar files verified" || { echo "Checksum failure when validating downloaded Laion moments"; exit 1; }
-
-## prompts and FID statistics from 30k image subset of coco2014
-## 33 MB
-$RCLONE config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
-$RCLONE copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/coco2014/val2014_30k.tsv ${COCO} -P
-
-$RCLONE config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
-$RCLONE copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/coco2014/val2014_30k_stats.npz ${COCO} -P
-
-# download checkpoints
-
-## clip (needed for text and vision encoders for validation)
-CLIP_WEIGHTS_URL="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin"
-CLIP_WEIGHTS_SHA256="9a78ef8e8c73fd0df621682e7a8e8eb36c6916cb3c16b291a082ecd52ab79cc4"
-CLIP_CONFIG_URL="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/raw/main/open_clip_config.json"
-wget -N -P ${CKPT}/clip ${CLIP_WEIGHTS_URL}
-wget -N -P ${CKPT}/clip ${CLIP_CONFIG_URL}
-echo "${CLIP_WEIGHTS_SHA256}  ${CKPT}/clip/open_clip_pytorch_model.bin"                    | sha256sum -c
-
-## sd (needed for latent->image decoder for validation, also has clip text encoder for training)
-SD_WEIGHTS_URL='https://huggingface.co/stabilityai/stable-diffusion-2-base/resolve/main/512-base-ema.ckpt'
-SD_WEIGHTS_SHA256="d635794c1fedfdfa261e065370bea59c651fc9bfa65dc6d67ad29e11869a1824"
-wget -N -P ${CKPT}/sd ${SD_WEIGHTS_URL}
-echo "${SD_WEIGHTS_SHA256}  ${CKPT}/sd/512-base-ema.ckpt"                    | sha256sum -c
-
-## inception (needed for validation)
-FID_WEIGHTS_URL='https://github.com/mseitzer/pytorch-fid/releases/download/fid_weights/pt_inception-2015-12-05-6726825d.pth'
-FID_WEIGHTS_SHA1="bd836944fd6db519dfd8d924aa457f5b3c8357ff"
-wget -N -P ${CKPT}/inception ${FID_WEIGHTS_URL}
-echo "${FID_WEIGHTS_SHA1}  ${CKPT}/inception/pt_inception-2015-12-05-6726825d.pth"                    | sha1sum -c

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_1xMI300X/dev_beam.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." AMD=1
-export MODEL="bert"
-export DEFAULT_FLOAT="HALF" GPUS=1 BS=128 EVAL_BS=128
-
-export BEAM=3 BEAM_UOPS_MAX=4000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
-export IGNORE_JIT_FIRST_BEAM=1
-# export BEAM_LOG_SURPASS_MAX=1
-# export BASEDIR="/raid/datasets/wiki"
-
-export RESET_STEP=1
-export BENCHMARK=10 BERT_LAYERS=2 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e  # Exit on any error
 
 export PYTHONPATH="." AMD=1
 export MODEL="bert"

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e  # Exit on any error
 
 export PYTHONPATH="." NV=1
 export MODEL="bert"

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_beam.sh

@@ -6,7 +6,7 @@ export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
 
 export FUSE_ARANGE=1 FUSE_ARANGE_UINT=0
 
-export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
+export BEAM=5 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
 export IGNORE_JIT_FIRST_BEAM=1
 export BEAM_LOG_SURPASS_MAX=1
 export BASEDIR="/raid/datasets/wiki"

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_run.sh

@@ -6,7 +6,7 @@ export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
 
 export FUSE_ARANGE=1 FUSE_ARANGE_UINT=0
 
-export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
+export BEAM=5 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
 export IGNORE_JIT_FIRST_BEAM=1
 export BASEDIR="/raid/datasets/wiki"
 

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e  # Exit on any error
 
 export PYTHONPATH="." AMD=1
 export MODEL="bert"

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/README.md

@@ -1,50 +0,0 @@
-# 1. Problem
-
-This problem uses the ResNet-50 CNN to do image classification.
-
-## Requirements
-
-Install tinygrad and mlperf-logging from master.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-### tinybox_red
-Disable cwsr
-This is the default on production tinybox red.
-```
-sudo vi /etc/modprobe.d/amdgpu.conf
-cat <<EOF > /etc/modprobe.d/amdgpu.conf
-options amdgpu cwsr_enable=0
-EOF
-sudo update-initramfs -u
-sudo reboot
-
-# validate
-sudo cat /sys/module/amdgpu/parameters/cwsr_enable #= 0
-```
-
-# 2. Directions
-
-## Steps to download and verify data
-
-```
-IMGNET_TRAIN=1 python3 extra/datasets/imagenet_download.py
-```
-
-## Steps for one time setup
-
-### tinybox_red
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh
-```
-
-## Steps to run benchmark
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh
-```

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_beam.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." NV=1
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-export BENCHMARK=10 DEBUG=2
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." NV=1
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-export EVAL_START_EPOCH=3 EVAL_FREQ=4
-
-export WANDB=1 PARALLEL=0
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_green/run_and_time.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-
-export PYTHONPATH="." NV=1
-export MODEL="resnet"
-export SUBMISSION_PLATFORM="tinybox_green"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=10 BEAM_PADTO=0
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=${LOGMLPERF:-1}
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="resnet_green_${DATETIME}_${SEED}.log"
-
-# init
-BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/README.md

@@ -1,50 +0,0 @@
-# 1. Problem
-
-This problem uses the ResNet-50 CNN to do image classification.
-
-## Requirements
-
-Install tinygrad and mlperf-logging from master.
-```
-git clone https://github.com/tinygrad/tinygrad.git
-python3 -m pip install -e ".[mlperf]"
-```
-
-### tinybox_green
-Install the p2p driver per [README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
-This is the default on production tinybox green.
-
-### tinybox_red
-Disable cwsr
-This is the default on production tinybox red.
-```
-sudo vi /etc/modprobe.d/amdgpu.conf
-cat <<EOF > /etc/modprobe.d/amdgpu.conf
-options amdgpu cwsr_enable=0
-EOF
-sudo update-initramfs -u
-sudo reboot
-
-# validate
-sudo cat /sys/module/amdgpu/parameters/cwsr_enable #= 0
-```
-
-# 2. Directions
-
-## Steps to download and verify data
-
-```
-IMGNET_TRAIN=1 python3 extra/datasets/imagenet_download.py
-```
-
-## Steps for one time setup
-
-### tinybox_red
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh
-```
-
-## Steps to run benchmark
-```
-examples/mlperf/training_submission_v4.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh
-```

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_beam.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." AMD=1
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export BENCHMARK=10 DEBUG=${DEBUG:-2}
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." AMD=1
-export MODEL="resnet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export EVAL_START_EPOCH=3 EVAL_FREQ=4
-
-export WANDB=1 PARALLEL=0
-
-python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/run_and_time.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e  # Exit on any error
-
-export PYTHONPATH="." AMD=1
-export MODEL="resnet"
-export SUBMISSION_PLATFORM="tinybox_red"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=1536 EVAL_BS=192
-
-export RESET_STEP=0
-
-export TRAIN_BEAM=4 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=2000 BEAM_UPCAST_MAX=96 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-# pip install -e ".[mlperf]"
-export LOGMLPERF=${LOGMLPERF:-1}
-
-export SEED=$RANDOM
-DATETIME=$(date "+%m%d%H%M")
-LOGFILE="resnet_red_${DATETIME}_${SEED}.log"
-
-# init
-sleep 5 && sudo rmmod amdgpu || true
-BENCHMARK=10 INITMLPERF=1 python3 examples/mlperf/model_train.py | tee $LOGFILE
-
-# run
-PARALLEL=0 RUNMLPERF=1 EVAL_START_EPOCH=3 EVAL_FREQ=4 python3 examples/mlperf/model_train.py | tee -a $LOGFILE

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/resnet/implementations/tinybox_red/setup.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-rocm-smi --setprofile compute
-rocm-smi --setmclk 3
-rocm-smi --setperflevel high
-
-# power cap to 350W
-echo "350000000" | sudo tee /sys/class/drm/card{1..6}/device/hwmon/hwmon*/power1_cap

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_green/run_and_time.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -e  # Exit on any error
 
 export PYTHONPATH="." NV=1
 export MODEL="retinanet"

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_beam.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." AMD=1
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export BENCHMARK=5 DEBUG=2
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/retinanet/implementations/tinybox_red/dev_run.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-export PYTHONPATH="." AMD=1
-export MODEL="retinanet"
-export DEFAULT_FLOAT="HALF" GPUS=6 BS=96 EVAL_BS=96
-export BASEDIR="/raid/datasets/openimages"
-
-# export RESET_STEP=0
-
-export TRAIN_BEAM=2 IGNORE_JIT_FIRST_BEAM=1 BEAM_UOPS_MAX=1500 BEAM_UPCAST_MAX=64 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 BEAM_PADTO=0
-
-export WANDB=1 PARALLEL=0
-export RUNMLPERF=1
-
-python examples/mlperf/model_train.py

examples/mlperf/training_submission_v5.0/tinycorp/benchmarks/stable_diffusion/implementations/tinybox_8xMI300X/dev_run.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-
-DATETIME=${2:-$(date "+%m%d%H%M")}
-LOGFILE="${HOME}/logs/sd_mi300x_${DATETIME}.log"
-# UNET_CKPTDIR must be set: training saves checkpoints to this path, then a separate eval process scans this path to know which checkpoints to eval
-export UNET_CKPTDIR="${HOME}/stable_diffusion/training_checkpoints/${DATETIME}"
-mkdir -p "${HOME}/logs" "$UNET_CKPTDIR"
-
-# run this script in isolation when using the --bg flag
-if [[ "${1:-}" == "--bg" ]]; then
-  echo "logging output to $LOGFILE"
-  echo "saving UNet checkpoints to $UNET_CKPTDIR"
-  script_path="$(readlink -f "${BASH_SOURCE[0]}")"
-  nohup bash "$script_path" run "$DATETIME" >"$LOGFILE" 2>&1 & disown $!
-  exit 0
-fi
-
-# venv management
-if [[ -d .venv-sd-mlperf ]]; then
-  . .venv-sd-mlperf/bin/activate
-else
-  python3 -m venv .venv-sd-mlperf && . .venv-sd-mlperf/bin/activate
-  pip install --index-url https://download.pytorch.org/whl/cpu torch && pip install tqdm numpy ftfy regex pillow scipy wandb webdataset
-fi
-pip list
-apt list --installed | grep amdgpu
-rocm-smi --version
-modinfo amdgpu | grep version
-
-export BEAM=2 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5 IGNORE_JIT_FIRST_BEAM=1 HCQDEV_WAIT_TIMEOUT_MS=300000
-export AMD_LLVM=0 # bf16 seems to require this
-export DATADIR="/raid/datasets/stable_diffusion"
-export CKPTDIR="/raid/weights/stable_diffusion"
-export EVAL_CKPT_DIR=$UNET_CKPTDIR
-export MODEL="stable_diffusion" PYTHONPATH="."
-export GPUS=8 BS=304
-export CONTEXT_BS=816 DENOISE_BS=600 DECODE_BS=384 INCEPTION_BS=560 CLIP_BS=240
-export WANDB=1
-export PARALLEL=4
-export PYTHONUNBUFFERED=1
-sudo rocm-smi -d 0 1 2 3 4 5 6 7 --setperfdeterminism 1500 || exit 1
-
-# Retry BEAM search if script fails before BEAM COMPLETE is printed, but don't retry after that
-run_retry(){ local try=0 max=5 code tmp py pgid kids
-  while :; do
-    tmp=$(mktemp)
-    setsid bash -c 'exec env "$@"' _ "$@" > >(tee -a "$LOGFILE" | tee "$tmp") 2>&1 &
-    py=$!; pgid=$(ps -o pgid= -p "$py" | tr -d ' ')
-    wait "$py"; code=$?
-    [[ -n "$pgid" ]] && { kill -TERM -"$pgid" 2>/dev/null; sleep 1; kill -KILL -"$pgid" 2>/dev/null; }
-    kids=$(pgrep -P "$py" || true)
-    while [[ -n "$kids" ]]; do
-      kill -TERM $kids 2>/dev/null; sleep 0.5
-      kids=$(for k in $kids; do pgrep -P "$k" || true; done)
-    done
-    grep -q 'BEAM COMPLETE' "$tmp" && { rm -f "$tmp"; return 1; }
-    rm -f "$tmp"
-    ((code==0)) && return 0
-    ((try>=max)) && return 2
-    ((try++)); sleep 90; echo "try = ${try}"
-  done
-}
-
-# Power limiting to 400W is only needed if GPUs fall out of sync (causing 2.2x increased train time) at higher power, which has been observed at 450W
-sudo rocm-smi -d 0 1 2 3 4 5 6 7 --setpoweroverdrive 750 && \
-run_retry TOTAL_CKPTS=7 python3 examples/mlperf/model_train.py; (( $? == 2 )) && { echo "training failed before BEAM completion"; exit 2; }
-sleep 90
-
-run_retry EVAL_SAMPLES=600 python3 examples/mlperf/model_eval.py; (( $? == 2 )) && { echo "eval failed before BEAM completion"; exit 2; }
-# Checkpoints will be evaluated in reverse chronological order, even if above training crashed early
-# STOP_IF_CONVERGED=1: Stop the eval after the first time convergence is detected; no more checkpoints will be evaluated after that.
-STOP_IF_CONVERGED=1 python3 examples/mlperf/model_eval.py

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_0.txt

@@ -0,0 +1,96 @@
+:::MLLOG {"namespace": "", "time_ms": 1745530925055, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925071, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925071, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925071, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925071, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925111, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745530925112, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745532610164, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745532618573, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745532618590, "event_type": "POINT_IN_TIME", "key": "seed", "value": 28597, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631107, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631108, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745532631109, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745532663595, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745533321994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745533363168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745533363168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37238641977310183, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.37238641977310183}}
+:::MLLOG {"namespace": "", "time_ms": 1745533638843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745533643497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745533643497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3876396149396896, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.3876396149396896}}
+:::MLLOG {"namespace": "", "time_ms": 1745533919587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745533924272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745533924272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4356461137533188, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.4356461137533188}}
+:::MLLOG {"namespace": "", "time_ms": 1745534200765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745534205421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745534205422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.523741614818573, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.523741614818573}}
+:::MLLOG {"namespace": "", "time_ms": 1745534481862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745534486501, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745534486501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6327109396457672, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6327109396457672}}
+:::MLLOG {"namespace": "", "time_ms": 1745534762710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745534767349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745534767349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6953712999820709, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.6953712999820709}}
+:::MLLOG {"namespace": "", "time_ms": 1745535043273, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745535047907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745535047907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7062251627445221, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7062251627445221}}
+:::MLLOG {"namespace": "", "time_ms": 1745535324088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745535328759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745535328760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7097240626811981, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7097240626811981}}
+:::MLLOG {"namespace": "", "time_ms": 1745535604822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745535609479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745535609479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7112975955009461, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7112975955009461}}
+:::MLLOG {"namespace": "", "time_ms": 1745535885084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745535889738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745535889738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.711781257390976, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.711781257390976}}
+:::MLLOG {"namespace": "", "time_ms": 1745536165354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745536170008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745536170009, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7138555943965912, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7138555943965912}}
+:::MLLOG {"namespace": "", "time_ms": 1745536445570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745536450183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745536450183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7145828008651733, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7145828008651733}}
+:::MLLOG {"namespace": "", "time_ms": 1745536725721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745536730380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745536730381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7148040235042572, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7148040235042572}}
+:::MLLOG {"namespace": "", "time_ms": 1745537006570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745537011215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745537011215, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7158548653125762, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7158548653125762}}
+:::MLLOG {"namespace": "", "time_ms": 1745537289257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745537293941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745537293941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160437941551209, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7160437941551209}}
+:::MLLOG {"namespace": "", "time_ms": 1745537580273, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745537584925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745537584926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7169912159442902, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7169912159442902}}
+:::MLLOG {"namespace": "", "time_ms": 1745537860132, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745537864770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745537864770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7176614046096802, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7176614046096802}}
+:::MLLOG {"namespace": "", "time_ms": 1745538142023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745538146643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745538146644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182221412658691, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7182221412658691}}
+:::MLLOG {"namespace": "", "time_ms": 1745538425213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745538429916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745538429916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186778724193573, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7186778724193573}}
+:::MLLOG {"namespace": "", "time_ms": 1745538705412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745538710041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745538710041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7190635979175568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7190635979175568}}
+:::MLLOG {"namespace": "", "time_ms": 1745538992202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745538996846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745538996846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7196638345718384, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.7196638345718384}}
+:::MLLOG {"namespace": "", "time_ms": 1745539272384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745539277037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745539277037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7201055765151978, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.7201055765151978}}
+:::MLLOG {"namespace": "", "time_ms": 1745539277038, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3289088}}
+:::MLLOG {"namespace": "", "time_ms": 1745539277038, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_1.txt

@@ -0,0 +1,87 @@
+:::MLLOG {"namespace": "", "time_ms": 1745539327854, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327869, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327870, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327870, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327870, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327912, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745539327912, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745540911909, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745540920744, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745540920760, "event_type": "POINT_IN_TIME", "key": "seed", "value": 15393, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933147, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933148, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745540933149, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745540962253, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745541620433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745541661192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745541661193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37288502752780917, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.37288502752780917}}
+:::MLLOG {"namespace": "", "time_ms": 1745541940327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745541944936, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745541944936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3860586792230606, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.3860586792230606}}
+:::MLLOG {"namespace": "", "time_ms": 1745542224659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745542229287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745542229287, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.429127961397171, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.429127961397171}}
+:::MLLOG {"namespace": "", "time_ms": 1745542510137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745542514771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745542514771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.529323160648346, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.529323160648346}}
+:::MLLOG {"namespace": "", "time_ms": 1745542796386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745542801034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745542801034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6631237685680389, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6631237685680389}}
+:::MLLOG {"namespace": "", "time_ms": 1745543082494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745543087128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745543087128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7024946749210358, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.7024946749210358}}
+:::MLLOG {"namespace": "", "time_ms": 1745543367964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745543372586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745543372587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.708392471075058, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.708392471075058}}
+:::MLLOG {"namespace": "", "time_ms": 1745543653532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745543658174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745543658174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.711252635717392, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.711252635717392}}
+:::MLLOG {"namespace": "", "time_ms": 1745543939454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745543944107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745543944107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7130857110023499, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7130857110023499}}
+:::MLLOG {"namespace": "", "time_ms": 1745544225356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745544230002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745544230002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7136700868606567, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7136700868606567}}
+:::MLLOG {"namespace": "", "time_ms": 1745544512872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745544517504, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745544517504, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7149688005447388, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7149688005447388}}
+:::MLLOG {"namespace": "", "time_ms": 1745544801071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745544805699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745544805700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.715419614315033, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.715419614315033}}
+:::MLLOG {"namespace": "", "time_ms": 1745545085900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745545090549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745545090549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160871505737305, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7160871505737305}}
+:::MLLOG {"namespace": "", "time_ms": 1745545375103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745545379758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745545379759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7170936107635498, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7170936107635498}}
+:::MLLOG {"namespace": "", "time_ms": 1745545660444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745545665097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745545665097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.717720341682434, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.717720341682434}}
+:::MLLOG {"namespace": "", "time_ms": 1745545946029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745545950685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745545950686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7183034479618072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7183034479618072}}
+:::MLLOG {"namespace": "", "time_ms": 1745546233082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745546237727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745546237727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182737410068512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7182737410068512}}
+:::MLLOG {"namespace": "", "time_ms": 1745546517916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745546522570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745546522571, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191624820232392, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7191624820232392}}
+:::MLLOG {"namespace": "", "time_ms": 1745546806258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745546810904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745546810904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200840294361115, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7200840294361115}}
+:::MLLOG {"namespace": "", "time_ms": 1745546810904, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2840576}}
+:::MLLOG {"namespace": "", "time_ms": 1745546810905, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_2.txt

@@ -0,0 +1,90 @@
+:::MLLOG {"namespace": "", "time_ms": 1745546859441, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859457, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859457, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859457, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859457, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859498, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745546859498, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745548472183, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745548480461, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745548480478, "event_type": "POINT_IN_TIME", "key": "seed", "value": 16206, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492724, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492725, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745548492726, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745548524140, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745549187391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745549229147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745549229147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3727019250392914, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.3727019250392914}}
+:::MLLOG {"namespace": "", "time_ms": 1745549513996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745549518578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745549518578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3877880424261093, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.3877880424261093}}
+:::MLLOG {"namespace": "", "time_ms": 1745549804473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745549809036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745549809036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4411086171865463, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.4411086171865463}}
+:::MLLOG {"namespace": "", "time_ms": 1745550093860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745550098426, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745550098426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5526047289371491, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5526047289371491}}
+:::MLLOG {"namespace": "", "time_ms": 1745550383034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745550387588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745550387589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6731186151504517, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6731186151504517}}
+:::MLLOG {"namespace": "", "time_ms": 1745550672196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745550676767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745550676767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7031940758228302, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.7031940758228302}}
+:::MLLOG {"namespace": "", "time_ms": 1745550961962, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745550966515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745550966515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7088134288787842, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7088134288787842}}
+:::MLLOG {"namespace": "", "time_ms": 1745551251075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745551255650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745551255650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7112709581851959, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7112709581851959}}
+:::MLLOG {"namespace": "", "time_ms": 1745551539957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745551544529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745551544530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7128494203090667, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7128494203090667}}
+:::MLLOG {"namespace": "", "time_ms": 1745551828589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745551833167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745551833167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7135251462459564, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7135251462459564}}
+:::MLLOG {"namespace": "", "time_ms": 1745552120068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745552124628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745552124628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147472620010376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7147472620010376}}
+:::MLLOG {"namespace": "", "time_ms": 1745552408212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745552412794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745552412794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7153081774711609, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7153081774711609}}
+:::MLLOG {"namespace": "", "time_ms": 1745552697710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745552702247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745552702247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7163431167602539, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7163431167602539}}
+:::MLLOG {"namespace": "", "time_ms": 1745552985842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745552990399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745552990399, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7166160047054291, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7166160047054291}}
+:::MLLOG {"namespace": "", "time_ms": 1745553277481, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745553282034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745553282035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7172852098941803, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7172852098941803}}
+:::MLLOG {"namespace": "", "time_ms": 1745553566684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745553571258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745553571259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7169495701789856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7169495701789856}}
+:::MLLOG {"namespace": "", "time_ms": 1745553854981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745553859558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745553859559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182259142398835, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7182259142398835}}
+:::MLLOG {"namespace": "", "time_ms": 1745554143398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745554147970, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745554147970, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718964672088623, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.718964672088623}}
+:::MLLOG {"namespace": "", "time_ms": 1745554435137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745554439700, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745554439700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7193355560302734, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7193355560302734}}
+:::MLLOG {"namespace": "", "time_ms": 1745554725252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745554729828, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745554729828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7201014399528504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7201014399528504}}
+:::MLLOG {"namespace": "", "time_ms": 1745554729829, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2990080}}
+:::MLLOG {"namespace": "", "time_ms": 1745554729829, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_3.txt

@@ -0,0 +1,93 @@
+:::MLLOG {"namespace": "", "time_ms": 1745554777762, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777778, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777778, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777778, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777778, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777819, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745554777819, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745556404094, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745556412199, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745556412215, "event_type": "POINT_IN_TIME", "key": "seed", "value": 20163, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424820, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424820, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424821, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745556424822, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745556457286, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745557115312, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745557156018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745557156018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3721550852060318, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.3721550852060318}}
+:::MLLOG {"namespace": "", "time_ms": 1745557435176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745557439784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745557439784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3927359789609909, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.3927359789609909}}
+:::MLLOG {"namespace": "", "time_ms": 1745557719263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745557723910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745557723910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.44803847670555114, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.44803847670555114}}
+:::MLLOG {"namespace": "", "time_ms": 1745558003706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745558008350, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745558008350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5591910660266877, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5591910660266877}}
+:::MLLOG {"namespace": "", "time_ms": 1745558288347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745558292987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745558292987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6610698461532593, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6610698461532593}}
+:::MLLOG {"namespace": "", "time_ms": 1745558573332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745558578000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745558578000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7006066799163818, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.7006066799163818}}
+:::MLLOG {"namespace": "", "time_ms": 1745558858774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745558863431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745558863431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7073053598403931, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7073053598403931}}
+:::MLLOG {"namespace": "", "time_ms": 1745559143329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745559147961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745559147962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7103267848491669, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7103267848491669}}
+:::MLLOG {"namespace": "", "time_ms": 1745559427612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745559432260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745559432260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7116473376750946, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7116473376750946}}
+:::MLLOG {"namespace": "", "time_ms": 1745559711261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745559715880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745559715881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7130024552345275, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7130024552345275}}
+:::MLLOG {"namespace": "", "time_ms": 1745559994602, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745559999235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745559999235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7140777409076691, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7140777409076691}}
+:::MLLOG {"namespace": "", "time_ms": 1745560278151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745560282795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745560282796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.714939397573471, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.714939397573471}}
+:::MLLOG {"namespace": "", "time_ms": 1745560561092, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745560565689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745560565690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7154468119144439, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7154468119144439}}
+:::MLLOG {"namespace": "", "time_ms": 1745560849874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745560854474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745560854474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161401331424713, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7161401331424713}}
+:::MLLOG {"namespace": "", "time_ms": 1745561132673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745561137319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745561137319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7163729786872863, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7163729786872863}}
+:::MLLOG {"namespace": "", "time_ms": 1745561418781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745561423392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745561423393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.717225980758667, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.717225980758667}}
+:::MLLOG {"namespace": "", "time_ms": 1745561701170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745561705798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745561705798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181021451950074, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7181021451950074}}
+:::MLLOG {"namespace": "", "time_ms": 1745561988084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745561992722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745561992722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191600739955902, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7191600739955902}}
+:::MLLOG {"namespace": "", "time_ms": 1745562273977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745562278658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745562278659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719371110200882, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.719371110200882}}
+:::MLLOG {"namespace": "", "time_ms": 1745562559865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745562564486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745562564486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718993628025055, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.718993628025055}}
+:::MLLOG {"namespace": "", "time_ms": 1745562845123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745562849755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745562849756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200249254703521, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.7200249254703521}}
+:::MLLOG {"namespace": "", "time_ms": 1745562849756, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3139584}}
+:::MLLOG {"namespace": "", "time_ms": 1745562849756, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_4.txt

@@ -0,0 +1,96 @@
+:::MLLOG {"namespace": "", "time_ms": 1745562899619, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899635, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899635, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899635, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899635, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899676, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745562899676, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745564587425, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745564596874, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745564596890, "event_type": "POINT_IN_TIME", "key": "seed", "value": 13352, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609291, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609292, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745564609293, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745564642816, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745565297838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745565339344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745565339345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3725235253572464, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.3725235253572464}}
+:::MLLOG {"namespace": "", "time_ms": 1745565617369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745565622024, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745565622025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.38939482867717745, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.38939482867717745}}
+:::MLLOG {"namespace": "", "time_ms": 1745565900583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745565905232, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745565905233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.43388367593288424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.43388367593288424}}
+:::MLLOG {"namespace": "", "time_ms": 1745566183301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745566187916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745566187917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5344274997711181, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5344274997711181}}
+:::MLLOG {"namespace": "", "time_ms": 1745566466251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745566470898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745566470898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6529988288879395, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6529988288879395}}
+:::MLLOG {"namespace": "", "time_ms": 1745566749556, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745566754223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745566754223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6987890124320983, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.6987890124320983}}
+:::MLLOG {"namespace": "", "time_ms": 1745567033420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745567038073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745567038073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7056617558002471, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7056617558002471}}
+:::MLLOG {"namespace": "", "time_ms": 1745567316228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745567320871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745567320871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7096020996570587, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7096020996570587}}
+:::MLLOG {"namespace": "", "time_ms": 1745567599127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745567603785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745567603785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7113405406475067, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7113405406475067}}
+:::MLLOG {"namespace": "", "time_ms": 1745567881894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745567886538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745567886539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7122747898101807, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7122747898101807}}
+:::MLLOG {"namespace": "", "time_ms": 1745568164600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745568169238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745568169239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7133678615093231, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7133678615093231}}
+:::MLLOG {"namespace": "", "time_ms": 1745568447131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745568451771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745568451771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7142562866210938, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7142562866210938}}
+:::MLLOG {"namespace": "", "time_ms": 1745568731625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745568736289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745568736290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7150763928890228, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7150763928890228}}
+:::MLLOG {"namespace": "", "time_ms": 1745569015501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745569020163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745569020163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7155498623847961, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7155498623847961}}
+:::MLLOG {"namespace": "", "time_ms": 1745569298438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745569303066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745569303066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160784900188446, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7160784900188446}}
+:::MLLOG {"namespace": "", "time_ms": 1745569581278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745569585933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745569585933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7170640766620636, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7170640766620636}}
+:::MLLOG {"namespace": "", "time_ms": 1745569866592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745569871241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745569871241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7173652410507202, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7173652410507202}}
+:::MLLOG {"namespace": "", "time_ms": 1745570152468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745570157114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745570157115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718361359834671, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.718361359834671}}
+:::MLLOG {"namespace": "", "time_ms": 1745570435938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745570440644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745570440644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186611413955688, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7186611413955688}}
+:::MLLOG {"namespace": "", "time_ms": 1745570726442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745570731082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745570731082, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191152453422547, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7191152453422547}}
+:::MLLOG {"namespace": "", "time_ms": 1745571012781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745571017427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745571017427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7195845186710358, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.7195845186710358}}
+:::MLLOG {"namespace": "", "time_ms": 1745571300015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745571304681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745571304682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200073778629303, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.7200073778629303}}
+:::MLLOG {"namespace": "", "time_ms": 1745571304682, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3289088}}
+:::MLLOG {"namespace": "", "time_ms": 1745571304682, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_5.txt

@@ -0,0 +1,96 @@
+:::MLLOG {"namespace": "", "time_ms": 1745571354572, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354587, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354587, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354587, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354587, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354629, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745571354629, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745573082356, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745573090559, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745573090575, "event_type": "POINT_IN_TIME", "key": "seed", "value": 29862, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103073, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103073, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103073, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103074, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103075, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103075, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103075, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745573103075, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745573134049, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745573811088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745573852243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745573852243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3729754567146301, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.3729754567146301}}
+:::MLLOG {"namespace": "", "time_ms": 1745574135466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745574140258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745574140258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.38697501420974734, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.38697501420974734}}
+:::MLLOG {"namespace": "", "time_ms": 1745574423983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745574428791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745574428791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.42467189133167266, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.42467189133167266}}
+:::MLLOG {"namespace": "", "time_ms": 1745574712426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745574717211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745574717212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5411350011825562, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5411350011825562}}
+:::MLLOG {"namespace": "", "time_ms": 1745575000437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745575005233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745575005233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6546417593955993, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6546417593955993}}
+:::MLLOG {"namespace": "", "time_ms": 1745575287460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745575292227, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745575292227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7013923466205597, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.7013923466205597}}
+:::MLLOG {"namespace": "", "time_ms": 1745575574252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745575579038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745575579038, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7082416176795959, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7082416176795959}}
+:::MLLOG {"namespace": "", "time_ms": 1745575860880, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745575865649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745575865649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7109045207500457, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7109045207500457}}
+:::MLLOG {"namespace": "", "time_ms": 1745576147355, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745576152107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745576152108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7121940672397613, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7121940672397613}}
+:::MLLOG {"namespace": "", "time_ms": 1745576433426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745576438176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745576438176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.713076388835907, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.713076388835907}}
+:::MLLOG {"namespace": "", "time_ms": 1745576719437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745576724186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745576724186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147435486316681, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7147435486316681}}
+:::MLLOG {"namespace": "", "time_ms": 1745577005513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745577010268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745577010268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7152598202228546, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7152598202228546}}
+:::MLLOG {"namespace": "", "time_ms": 1745577294417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745577299176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745577299177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7158841907978057, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7158841907978057}}
+:::MLLOG {"namespace": "", "time_ms": 1745577583196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745577587938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745577587938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7162841558456421, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7162841558456421}}
+:::MLLOG {"namespace": "", "time_ms": 1745577872125, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745577876872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745577876872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171514630317688, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7171514630317688}}
+:::MLLOG {"namespace": "", "time_ms": 1745578157392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745578162167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745578162167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.717557144165039, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.717557144165039}}
+:::MLLOG {"namespace": "", "time_ms": 1745578442695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745578447477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745578447478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181805670261383, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7181805670261383}}
+:::MLLOG {"namespace": "", "time_ms": 1745578732198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745578736960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745578736960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7193208992481231, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7193208992481231}}
+:::MLLOG {"namespace": "", "time_ms": 1745579017075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745579021830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745579021831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7190930187702179, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7190930187702179}}
+:::MLLOG {"namespace": "", "time_ms": 1745579308467, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745579313199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745579313199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7193685650825501, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7193685650825501}}
+:::MLLOG {"namespace": "", "time_ms": 1745579595974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745579600728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745579600728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7198175132274628, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.7198175132274628}}
+:::MLLOG {"namespace": "", "time_ms": 1745579881098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745579885849, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745579885849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.720059609413147, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.720059609413147}}
+:::MLLOG {"namespace": "", "time_ms": 1745579885849, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3289088}}
+:::MLLOG {"namespace": "", "time_ms": 1745579885849, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_6.txt

@@ -0,0 +1,108 @@
+:::MLLOG {"namespace": "", "time_ms": 1745579935971, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745579935986, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745579935987, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745579935987, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745579935987, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745579936029, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745579936029, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745581719259, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745581728446, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745581728462, "event_type": "POINT_IN_TIME", "key": "seed", "value": 7754, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740992, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740993, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745581740994, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745581773454, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745582423629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745582464633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745582464633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37272038757801057, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.37272038757801057}}
+:::MLLOG {"namespace": "", "time_ms": 1745582740508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745582745067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745582745068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.38925057649612427, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.38925057649612427}}
+:::MLLOG {"namespace": "", "time_ms": 1745583021215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745583025776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745583025776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.43427990674972533, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.43427990674972533}}
+:::MLLOG {"namespace": "", "time_ms": 1745583301981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745583306560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745583306561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5278923571109772, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5278923571109772}}
+:::MLLOG {"namespace": "", "time_ms": 1745583582767, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745583587345, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745583587345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6390926957130432, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6390926957130432}}
+:::MLLOG {"namespace": "", "time_ms": 1745583863550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745583868135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745583868136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6956972718238831, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.6956972718238831}}
+:::MLLOG {"namespace": "", "time_ms": 1745584144322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745584148929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745584148929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.705748564004898, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.705748564004898}}
+:::MLLOG {"namespace": "", "time_ms": 1745584425116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745584429704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745584429704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7098638355731964, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7098638355731964}}
+:::MLLOG {"namespace": "", "time_ms": 1745584705931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745584710512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745584710512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7110268771648407, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7110268771648407}}
+:::MLLOG {"namespace": "", "time_ms": 1745584986656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745584991214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745584991214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.712305212020874, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.712305212020874}}
+:::MLLOG {"namespace": "", "time_ms": 1745585267330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745585271920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745585271921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7135731339454651, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7135731339454651}}
+:::MLLOG {"namespace": "", "time_ms": 1745585551669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745585556247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745585556248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7142563343048096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7142563343048096}}
+:::MLLOG {"namespace": "", "time_ms": 1745585833849, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745585838410, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745585838410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7152255415916443, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7152255415916443}}
+:::MLLOG {"namespace": "", "time_ms": 1745586114185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745586118755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745586118755, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7159430027008057, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7159430027008057}}
+:::MLLOG {"namespace": "", "time_ms": 1745586394477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745586399048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745586399048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7162509083747863, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7162509083747863}}
+:::MLLOG {"namespace": "", "time_ms": 1745586674813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745586679413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745586679414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7167587816715241, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7167587816715241}}
+:::MLLOG {"namespace": "", "time_ms": 1745586958455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745586963029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745586963029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7172225117683411, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7172225117683411}}
+:::MLLOG {"namespace": "", "time_ms": 1745587244889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745587249485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745587249485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7183384358882904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7183384358882904}}
+:::MLLOG {"namespace": "", "time_ms": 1745587524929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745587529566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745587529566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180015325546265, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7180015325546265}}
+:::MLLOG {"namespace": "", "time_ms": 1745587808591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745587813157, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745587813158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7184844195842743, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7184844195842743}}
+:::MLLOG {"namespace": "", "time_ms": 1745588093001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745588097586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745588097587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718977439403534, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.718977439403534}}
+:::MLLOG {"namespace": "", "time_ms": 1745588375341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745588379922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745588379922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7192779302597045, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.7192779302597045}}
+:::MLLOG {"namespace": "", "time_ms": 1745588655781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3438592, "step_num": 3358}}
+:::MLLOG {"namespace": "", "time_ms": 1745588660364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3438592, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3438592, "step_num": 3358, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745588660364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719474196434021, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3438592, "masked_lm_accuracy": 0.719474196434021}}
+:::MLLOG {"namespace": "", "time_ms": 1745588936187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3588096, "step_num": 3504}}
+:::MLLOG {"namespace": "", "time_ms": 1745588940778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3588096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3588096, "step_num": 3504, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745588940778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7198640763759613, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3588096, "masked_lm_accuracy": 0.7198640763759613}}
+:::MLLOG {"namespace": "", "time_ms": 1745589216640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3737600, "step_num": 3650}}
+:::MLLOG {"namespace": "", "time_ms": 1745589221228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3737600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3737600, "step_num": 3650, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745589221228, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7199562847614288, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3737600, "masked_lm_accuracy": 0.7199562847614288}}
+:::MLLOG {"namespace": "", "time_ms": 1745589497055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3887104, "step_num": 3796}}
+:::MLLOG {"namespace": "", "time_ms": 1745589501645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3887104, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3887104, "step_num": 3796, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745589501646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7206668496131897, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3887104, "masked_lm_accuracy": 0.7206668496131897}}
+:::MLLOG {"namespace": "", "time_ms": 1745589501696, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3887104, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3887104}}
+:::MLLOG {"namespace": "", "time_ms": 1745589501696, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_7.txt

@@ -0,0 +1,105 @@
+:::MLLOG {"namespace": "", "time_ms": 1745589550561, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550577, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550577, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550577, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550577, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550621, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745589550622, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745591179400, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745591187432, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745591187448, "event_type": "POINT_IN_TIME", "key": "seed", "value": 25385, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199679, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199680, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745591199681, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745591230921, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745591901120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745591941186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745591941187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37098502218723295, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.37098502218723295}}
+:::MLLOG {"namespace": "", "time_ms": 1745592221446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745592226014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745592226015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.38992435932159425, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.38992435932159425}}
+:::MLLOG {"namespace": "", "time_ms": 1745592506825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745592511386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745592511386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4360688954591751, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.4360688954591751}}
+:::MLLOG {"namespace": "", "time_ms": 1745592792597, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745592797166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745592797167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5389649093151092, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5389649093151092}}
+:::MLLOG {"namespace": "", "time_ms": 1745593078507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745593083063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745593083064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6623023927211762, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6623023927211762}}
+:::MLLOG {"namespace": "", "time_ms": 1745593364383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745593368960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745593368960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6989677131175995, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.6989677131175995}}
+:::MLLOG {"namespace": "", "time_ms": 1745593650221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745593654771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745593654771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7055331587791442, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7055331587791442}}
+:::MLLOG {"namespace": "", "time_ms": 1745593935981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745593940552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745593940552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7088817238807679, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7088817238807679}}
+:::MLLOG {"namespace": "", "time_ms": 1745594221979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745594226554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745594226554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7105159401893616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7105159401893616}}
+:::MLLOG {"namespace": "", "time_ms": 1745594507867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745594512457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745594512457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7124811410903931, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7124811410903931}}
+:::MLLOG {"namespace": "", "time_ms": 1745594793979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745594798544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745594798545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7134620428085328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7134620428085328}}
+:::MLLOG {"namespace": "", "time_ms": 1745595079851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745595084445, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745595084445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7141174793243408, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7141174793243408}}
+:::MLLOG {"namespace": "", "time_ms": 1745595368793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745595373357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745595373357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7149844408035279, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7149844408035279}}
+:::MLLOG {"namespace": "", "time_ms": 1745595656391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745595660943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745595660944, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7157506585121155, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7157506585121155}}
+:::MLLOG {"namespace": "", "time_ms": 1745595946346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745595950902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745595950902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160208821296692, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7160208821296692}}
+:::MLLOG {"namespace": "", "time_ms": 1745596234396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745596238968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745596238968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7169745147228241, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7169745147228241}}
+:::MLLOG {"namespace": "", "time_ms": 1745596519214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745596523783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745596523784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7170299768447876, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7170299768447876}}
+:::MLLOG {"namespace": "", "time_ms": 1745596804022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745596808588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745596808589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186152815818787, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7186152815818787}}
+:::MLLOG {"namespace": "", "time_ms": 1745597088735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745597093297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745597093297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7184043228626251, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7184043228626251}}
+:::MLLOG {"namespace": "", "time_ms": 1745597376858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745597381430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745597381430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186801016330719, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7186801016330719}}
+:::MLLOG {"namespace": "", "time_ms": 1745597661697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745597666257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745597666257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191139340400696, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.7191139340400696}}
+:::MLLOG {"namespace": "", "time_ms": 1745597949669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745597954226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745597954226, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7192700445652008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.7192700445652008}}
+:::MLLOG {"namespace": "", "time_ms": 1745598236392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3438592, "step_num": 3358}}
+:::MLLOG {"namespace": "", "time_ms": 1745598240947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3438592, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3438592, "step_num": 3358, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745598240948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7196909546852112, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3438592, "masked_lm_accuracy": 0.7196909546852112}}
+:::MLLOG {"namespace": "", "time_ms": 1745598521853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3588096, "step_num": 3504}}
+:::MLLOG {"namespace": "", "time_ms": 1745598526430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3588096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3588096, "step_num": 3504, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745598526431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7199203789234161, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3588096, "masked_lm_accuracy": 0.7199203789234161}}
+:::MLLOG {"namespace": "", "time_ms": 1745598807367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3737600, "step_num": 3650}}
+:::MLLOG {"namespace": "", "time_ms": 1745598811951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3737600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3737600, "step_num": 3650, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745598811951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200918376445771, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3737600, "masked_lm_accuracy": 0.7200918376445771}}
+:::MLLOG {"namespace": "", "time_ms": 1745598811952, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3737600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3737600}}
+:::MLLOG {"namespace": "", "time_ms": 1745598811952, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_8.txt

@@ -0,0 +1,102 @@
+:::MLLOG {"namespace": "", "time_ms": 1745598860595, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860610, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860610, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860610, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860610, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860653, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745598860653, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745600478970, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745600487253, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745600487271, "event_type": "POINT_IN_TIME", "key": "seed", "value": 32629, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500186, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500186, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500186, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500186, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500187, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500188, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745600500188, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745600532785, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745601205060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745601247392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745601247392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3731185048818588, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.3731185048818588}}
+:::MLLOG {"namespace": "", "time_ms": 1745601525682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745601530299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745601530300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.39272683262825014, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.39272683262825014}}
+:::MLLOG {"namespace": "", "time_ms": 1745601809277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745601813902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745601813902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.43582180738449094, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.43582180738449094}}
+:::MLLOG {"namespace": "", "time_ms": 1745602092985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745602097585, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745602097585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5279603898525238, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.5279603898525238}}
+:::MLLOG {"namespace": "", "time_ms": 1745602376585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745602381222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745602381222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6447000563144684, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.6447000563144684}}
+:::MLLOG {"namespace": "", "time_ms": 1745602660370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745602664997, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745602664997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6944801509380341, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.6944801509380341}}
+:::MLLOG {"namespace": "", "time_ms": 1745602944009, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745602948636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745602948636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7051402449607849, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7051402449607849}}
+:::MLLOG {"namespace": "", "time_ms": 1745603227727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745603232331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745603232331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7089028596878052, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7089028596878052}}
+:::MLLOG {"namespace": "", "time_ms": 1745603511699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745603516332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745603516332, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7109102308750153, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7109102308750153}}
+:::MLLOG {"namespace": "", "time_ms": 1745603795809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745603800431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745603800432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7122457921504974, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7122457921504974}}
+:::MLLOG {"namespace": "", "time_ms": 1745604080028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745604084645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745604084645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7131429493427277, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7131429493427277}}
+:::MLLOG {"namespace": "", "time_ms": 1745604364612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745604369261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745604369261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7144218623638153, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7144218623638153}}
+:::MLLOG {"namespace": "", "time_ms": 1745604649086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745604653711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745604653711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7148768424987793, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7148768424987793}}
+:::MLLOG {"namespace": "", "time_ms": 1745604935309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745604939938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745604939938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7154791951179504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.7154791951179504}}
+:::MLLOG {"namespace": "", "time_ms": 1745605219927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745605224572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745605224572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161833882331848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7161833882331848}}
+:::MLLOG {"namespace": "", "time_ms": 1745605510499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745605515142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745605515142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7165493428707123, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7165493428707123}}
+:::MLLOG {"namespace": "", "time_ms": 1745605794599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745605799240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745605799241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7170698583126068, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7170698583126068}}
+:::MLLOG {"namespace": "", "time_ms": 1745606082218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745606086859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745606086860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182413637638092, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7182413637638092}}
+:::MLLOG {"namespace": "", "time_ms": 1745606370703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2840576, "step_num": 2774}}
+:::MLLOG {"namespace": "", "time_ms": 1745606375349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2840576, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2840576, "step_num": 2774, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745606375350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7183951079845429, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2840576, "masked_lm_accuracy": 0.7183951079845429}}
+:::MLLOG {"namespace": "", "time_ms": 1745606655495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2990080, "step_num": 2920}}
+:::MLLOG {"namespace": "", "time_ms": 1745606660147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2990080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2990080, "step_num": 2920, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745606660147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185700833797455, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2990080, "masked_lm_accuracy": 0.7185700833797455}}
+:::MLLOG {"namespace": "", "time_ms": 1745606941853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3139584, "step_num": 3066}}
+:::MLLOG {"namespace": "", "time_ms": 1745606946474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3139584, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3139584, "step_num": 3066, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745606946474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719360601902008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3139584, "masked_lm_accuracy": 0.719360601902008}}
+:::MLLOG {"namespace": "", "time_ms": 1745607229955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3289088, "step_num": 3212}}
+:::MLLOG {"namespace": "", "time_ms": 1745607234597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3289088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3289088, "step_num": 3212, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745607234597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7198850989341736, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3289088, "masked_lm_accuracy": 0.7198850989341736}}
+:::MLLOG {"namespace": "", "time_ms": 1745607515289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3438592, "step_num": 3358}}
+:::MLLOG {"namespace": "", "time_ms": 1745607519928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3438592, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3438592, "step_num": 3358, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745607519928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7198968529701233, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3438592, "masked_lm_accuracy": 0.7198968529701233}}
+:::MLLOG {"namespace": "", "time_ms": 1745607800552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3588096, "step_num": 3504}}
+:::MLLOG {"namespace": "", "time_ms": 1745607805193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3588096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3588096, "step_num": 3504, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745607805194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7202662408351899, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3588096, "masked_lm_accuracy": 0.7202662408351899}}
+:::MLLOG {"namespace": "", "time_ms": 1745607805194, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3588096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3588096}}
+:::MLLOG {"namespace": "", "time_ms": 1745607805194, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_8xMI300X/bert/result_9.txt

@@ -0,0 +1,84 @@
+:::MLLOG {"namespace": "", "time_ms": 1745607853432, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853448, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_8xMI300X", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853448, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853448, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853448, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853489, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745607853490, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745609448260, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745609456393, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745609456411, "event_type": "POINT_IN_TIME", "key": "seed", "value": 24956, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468886, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 1024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468886, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468886, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468886, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.0011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.60466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.85437, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468887, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 3900, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468888, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468888, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10240, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745609468888, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3993600, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745609497729, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745610146211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149504, "step_num": 146}}
+:::MLLOG {"namespace": "", "time_ms": 1745610186645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149504, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149504, "step_num": 146, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745610186646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37167071998119355, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149504, "masked_lm_accuracy": 0.37167071998119355}}
+:::MLLOG {"namespace": "", "time_ms": 1745610462149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299008, "step_num": 292}}
+:::MLLOG {"namespace": "", "time_ms": 1745610466711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299008, "step_num": 292, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745610466711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3860401749610901, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299008, "masked_lm_accuracy": 0.3860401749610901}}
+:::MLLOG {"namespace": "", "time_ms": 1745610742915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 448512, "step_num": 438}}
+:::MLLOG {"namespace": "", "time_ms": 1745610747488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 448512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 448512, "step_num": 438, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745610747488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5218028664588928, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 448512, "masked_lm_accuracy": 0.5218028664588928}}
+:::MLLOG {"namespace": "", "time_ms": 1745611023698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 598016, "step_num": 584}}
+:::MLLOG {"namespace": "", "time_ms": 1745611028274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 598016, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 598016, "step_num": 584, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745611028275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.673934280872345, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 598016, "masked_lm_accuracy": 0.673934280872345}}
+:::MLLOG {"namespace": "", "time_ms": 1745611304381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 747520, "step_num": 730}}
+:::MLLOG {"namespace": "", "time_ms": 1745611308983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 747520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 747520, "step_num": 730, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745611308983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7041513025760651, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 747520, "masked_lm_accuracy": 0.7041513025760651}}
+:::MLLOG {"namespace": "", "time_ms": 1745611585202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 897024, "step_num": 876}}
+:::MLLOG {"namespace": "", "time_ms": 1745611589795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 897024, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 897024, "step_num": 876, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745611589795, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7099382877349854, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 897024, "masked_lm_accuracy": 0.7099382877349854}}
+:::MLLOG {"namespace": "", "time_ms": 1745611866035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1046528, "step_num": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745611870610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1046528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1046528, "step_num": 1022, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745611870610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7122330486774444, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1046528, "masked_lm_accuracy": 0.7122330486774444}}
+:::MLLOG {"namespace": "", "time_ms": 1745612146933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1196032, "step_num": 1168}}
+:::MLLOG {"namespace": "", "time_ms": 1745612151528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1196032, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1196032, "step_num": 1168, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745612151529, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7136648654937744, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1196032, "masked_lm_accuracy": 0.7136648654937744}}
+:::MLLOG {"namespace": "", "time_ms": 1745612428104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1345536, "step_num": 1314}}
+:::MLLOG {"namespace": "", "time_ms": 1745612432695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1345536, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1345536, "step_num": 1314, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745612432695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7144280135631561, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1345536, "masked_lm_accuracy": 0.7144280135631561}}
+:::MLLOG {"namespace": "", "time_ms": 1745612709067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1495040, "step_num": 1460}}
+:::MLLOG {"namespace": "", "time_ms": 1745612713674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1495040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1495040, "step_num": 1460, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745612713675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147453427314758, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1495040, "masked_lm_accuracy": 0.7147453427314758}}
+:::MLLOG {"namespace": "", "time_ms": 1745612989926, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1644544, "step_num": 1606}}
+:::MLLOG {"namespace": "", "time_ms": 1745612994505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1644544, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1644544, "step_num": 1606, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745612994505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7162890911102295, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1644544, "masked_lm_accuracy": 0.7162890911102295}}
+:::MLLOG {"namespace": "", "time_ms": 1745613274460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1794048, "step_num": 1752}}
+:::MLLOG {"namespace": "", "time_ms": 1745613279032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1794048, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1794048, "step_num": 1752, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745613279032, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7166337609291077, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1794048, "masked_lm_accuracy": 0.7166337609291077}}
+:::MLLOG {"namespace": "", "time_ms": 1745613554938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1943552, "step_num": 1898}}
+:::MLLOG {"namespace": "", "time_ms": 1745613559537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1943552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1943552, "step_num": 1898, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745613559537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7172951698303223, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1943552, "masked_lm_accuracy": 0.7172951698303223}}
+:::MLLOG {"namespace": "", "time_ms": 1745613842482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2093056, "step_num": 2044}}
+:::MLLOG {"namespace": "", "time_ms": 1745613847063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2093056, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2093056, "step_num": 2044, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745613847063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718510490655899, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2093056, "masked_lm_accuracy": 0.718510490655899}}
+:::MLLOG {"namespace": "", "time_ms": 1745614126049, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2242560, "step_num": 2190}}
+:::MLLOG {"namespace": "", "time_ms": 1745614130639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2242560, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2242560, "step_num": 2190, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745614130639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7184054613113403, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2242560, "masked_lm_accuracy": 0.7184054613113403}}
+:::MLLOG {"namespace": "", "time_ms": 1745614406167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2392064, "step_num": 2336}}
+:::MLLOG {"namespace": "", "time_ms": 1745614410769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2392064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2392064, "step_num": 2336, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745614410769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7189517140388488, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2392064, "masked_lm_accuracy": 0.7189517140388488}}
+:::MLLOG {"namespace": "", "time_ms": 1745614686010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2541568, "step_num": 2482}}
+:::MLLOG {"namespace": "", "time_ms": 1745614690621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2541568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2541568, "step_num": 2482, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745614690622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191794335842132, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2541568, "masked_lm_accuracy": 0.7191794335842132}}
+:::MLLOG {"namespace": "", "time_ms": 1745614968159, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2691072, "step_num": 2628}}
+:::MLLOG {"namespace": "", "time_ms": 1745614972735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2691072, "step_num": 2628, "samples_count": 10240}}
+:::MLLOG {"namespace": "", "time_ms": 1745614972736, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7202518999576568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2691072, "masked_lm_accuracy": 0.7202518999576568}}
+:::MLLOG {"namespace": "", "time_ms": 1745614972736, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2691072, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2691072}}
+:::MLLOG {"namespace": "", "time_ms": 1745614972736, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_0.txt

@@ -0,0 +1,96 @@
+:::MLLOG {"namespace": "", "time_ms": 1745542295813, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745542295826, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745542295827, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745542295827, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745542295827, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745542296498, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745542296499, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745543895182, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745543905464, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745543905478, "event_type": "POINT_IN_TIME", "key": "seed", "value": 13785, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921082, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921082, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921082, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745543921084, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745543978728, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745544850167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745544889032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745544889032, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37353726426760353, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37353726426760353}}
+:::MLLOG {"namespace": "", "time_ms": 1745545487519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745545498684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745545498684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.41237829072134835, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.41237829072134835}}
+:::MLLOG {"namespace": "", "time_ms": 1745546090548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745546100134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745546100135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034303157102494, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.5034303157102494}}
+:::MLLOG {"namespace": "", "time_ms": 1745546693196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745546702828, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745546702828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6373415260087876, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6373415260087876}}
+:::MLLOG {"namespace": "", "time_ms": 1745547295519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745547305142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745547305142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6955064086686997, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.6955064086686997}}
+:::MLLOG {"namespace": "", "time_ms": 1745547896140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745547905833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745547905833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7047913125583104, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7047913125583104}}
+:::MLLOG {"namespace": "", "time_ms": 1745548498633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745548508143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745548508143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7078574929918562, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7078574929918562}}
+:::MLLOG {"namespace": "", "time_ms": 1745549101227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745549110866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745549110866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7101279389290582, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7101279389290582}}
+:::MLLOG {"namespace": "", "time_ms": 1745549701586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745549711083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745549711084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7115791564895994, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7115791564895994}}
+:::MLLOG {"namespace": "", "time_ms": 1745550303748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745550313288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745550313288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.713118288630531, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.713118288630531}}
+:::MLLOG {"namespace": "", "time_ms": 1745550906395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745550916128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745550916128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7131774215471177, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7131774215471177}}
+:::MLLOG {"namespace": "", "time_ms": 1745551506889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745551516739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745551516739, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7145353862217494, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7145353862217494}}
+:::MLLOG {"namespace": "", "time_ms": 1745552113873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745552123576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745552123576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7154489999725705, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7154489999725705}}
+:::MLLOG {"namespace": "", "time_ms": 1745552716142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745552725770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745552725771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7163924120721363, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7163924120721363}}
+:::MLLOG {"namespace": "", "time_ms": 1745553324307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745553333814, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745553333815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.716705474399385, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.716705474399385}}
+:::MLLOG {"namespace": "", "time_ms": 1745553926317, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745553936094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745553936094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7176795959472656, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7176795959472656}}
+:::MLLOG {"namespace": "", "time_ms": 1745554531890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745554542845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745554542846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7178676713080633, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7178676713080633}}
+:::MLLOG {"namespace": "", "time_ms": 1745555133528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745555143101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745555143101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7184804496311006, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7184804496311006}}
+:::MLLOG {"namespace": "", "time_ms": 1745555747973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745555757481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745555757481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718643741948264, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.718643741948264}}
+:::MLLOG {"namespace": "", "time_ms": 1745556357940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745556367552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745556367552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7190265468188695, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7190265468188695}}
+:::MLLOG {"namespace": "", "time_ms": 1745556962657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3148992, "step_num": 32802}}
+:::MLLOG {"namespace": "", "time_ms": 1745556972166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3148992, "step_num": 32802, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745556972166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7194553653399149, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3148992, "masked_lm_accuracy": 0.7194553653399149}}
+:::MLLOG {"namespace": "", "time_ms": 1745557564941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3298944, "step_num": 34364}}
+:::MLLOG {"namespace": "", "time_ms": 1745557574552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3298944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3298944, "step_num": 34364, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745557574552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.720384947458903, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3298944, "masked_lm_accuracy": 0.720384947458903}}
+:::MLLOG {"namespace": "", "time_ms": 1745557574552, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3298944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3298944}}
+:::MLLOG {"namespace": "", "time_ms": 1745557574552, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_1.txt

@@ -0,0 +1,96 @@
+:::MLLOG {"namespace": "", "time_ms": 1745557594256, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594270, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594270, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594270, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594270, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594941, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745557594941, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745559214897, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745559225413, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745559225427, "event_type": "POINT_IN_TIME", "key": "seed", "value": 32312, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240479, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240480, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240481, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240482, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745559240482, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745559294576, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745560166054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745560205220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745560205220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37391537456285384, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37391537456285384}}
+:::MLLOG {"namespace": "", "time_ms": 1745560805893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745560815463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745560815463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4109735344137464, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4109735344137464}}
+:::MLLOG {"namespace": "", "time_ms": 1745561410351, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745561419975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745561419975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49219877805028645, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.49219877805028645}}
+:::MLLOG {"namespace": "", "time_ms": 1745562014334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745562023781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745562023781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6128720873878115, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6128720873878115}}
+:::MLLOG {"namespace": "", "time_ms": 1745562618009, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745562627536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745562627537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.697400647117978, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.697400647117978}}
+:::MLLOG {"namespace": "", "time_ms": 1745563220282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745563229809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745563229810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.705579203651065, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.705579203651065}}
+:::MLLOG {"namespace": "", "time_ms": 1745563823608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745563833062, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745563833062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7082960753213792, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7082960753213792}}
+:::MLLOG {"namespace": "", "time_ms": 1745564427147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745564436605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745564436605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7107288156236921, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7107288156236921}}
+:::MLLOG {"namespace": "", "time_ms": 1745565028811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745565038436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745565038436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7119181950887045, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7119181950887045}}
+:::MLLOG {"namespace": "", "time_ms": 1745565632265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745565641826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745565641827, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7127573898860387, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7127573898860387}}
+:::MLLOG {"namespace": "", "time_ms": 1745566234178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745566245324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745566245325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7137119378362383, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7137119378362383}}
+:::MLLOG {"namespace": "", "time_ms": 1745566837891, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745566847554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745566847554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7144972681999207, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7144972681999207}}
+:::MLLOG {"namespace": "", "time_ms": 1745567441551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745567451141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745567451141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7152219454447428, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7152219454447428}}
+:::MLLOG {"namespace": "", "time_ms": 1745568051665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745568061105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745568061106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7158037361644564, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7158037361644564}}
+:::MLLOG {"namespace": "", "time_ms": 1745568652783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745568662536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745568662536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160673771585737, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7160673771585737}}
+:::MLLOG {"namespace": "", "time_ms": 1745569255983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745569265537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745569265537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.716984482606252, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.716984482606252}}
+:::MLLOG {"namespace": "", "time_ms": 1745569865307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745569874760, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745569874760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171691741262164, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7171691741262164}}
+:::MLLOG {"namespace": "", "time_ms": 1745570473145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745570482762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745570482763, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180932062012809, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7180932062012809}}
+:::MLLOG {"namespace": "", "time_ms": 1745571082302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745571091949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745571091950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185394008954366, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.7185394008954366}}
+:::MLLOG {"namespace": "", "time_ms": 1745571701179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745571710616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745571710616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718915491444724, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.718915491444724}}
+:::MLLOG {"namespace": "", "time_ms": 1745572308022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3148992, "step_num": 32802}}
+:::MLLOG {"namespace": "", "time_ms": 1745572317478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3148992, "step_num": 32802, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745572317479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7197886319387526, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3148992, "masked_lm_accuracy": 0.7197886319387526}}
+:::MLLOG {"namespace": "", "time_ms": 1745572911332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3298944, "step_num": 34364}}
+:::MLLOG {"namespace": "", "time_ms": 1745572920792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3298944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3298944, "step_num": 34364, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745572920792, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7201146880785624, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3298944, "masked_lm_accuracy": 0.7201146880785624}}
+:::MLLOG {"namespace": "", "time_ms": 1745572920793, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3298944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3298944}}
+:::MLLOG {"namespace": "", "time_ms": 1745572920793, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_2.txt

@@ -0,0 +1,93 @@
+:::MLLOG {"namespace": "", "time_ms": 1745572941616, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745572941629, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745572941630, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745572941630, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745572941630, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745572942291, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745572942291, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745574879961, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745574890440, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745574890454, "event_type": "POINT_IN_TIME", "key": "seed", "value": 6986, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906135, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906135, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906135, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906135, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906135, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906136, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906137, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906137, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906137, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745574906137, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745574955636, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745575830554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745575870376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745575870377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3737282611074902, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.3737282611074902}}
+:::MLLOG {"namespace": "", "time_ms": 1745576471845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745576481412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745576481413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4033156321162269, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4033156321162269}}
+:::MLLOG {"namespace": "", "time_ms": 1745577076023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745577085523, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745577085523, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4883529495625269, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.4883529495625269}}
+:::MLLOG {"namespace": "", "time_ms": 1745577681373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745577691036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745577691036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6172424344789414, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6172424344789414}}
+:::MLLOG {"namespace": "", "time_ms": 1745578286633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745578296292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745578296292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6984787407375518, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.6984787407375518}}
+:::MLLOG {"namespace": "", "time_ms": 1745578891885, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745578901678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745578901679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7075154764311654, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7075154764311654}}
+:::MLLOG {"namespace": "", "time_ms": 1745579495449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745579505011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745579505011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7099487849644253, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7099487849644253}}
+:::MLLOG {"namespace": "", "time_ms": 1745580100680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745580110198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745580110198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7115290715580895, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7115290715580895}}
+:::MLLOG {"namespace": "", "time_ms": 1745580703932, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745580713443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745580713443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7128627572740828, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7128627572740828}}
+:::MLLOG {"namespace": "", "time_ms": 1745581308511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745581318215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745581318215, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7135378882998512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7135378882998512}}
+:::MLLOG {"namespace": "", "time_ms": 1745581913204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745581922724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745581922724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7148078146434965, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7148078146434965}}
+:::MLLOG {"namespace": "", "time_ms": 1745582522356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745582531872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745582531872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7152938303493318, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7152938303493318}}
+:::MLLOG {"namespace": "", "time_ms": 1745583125261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745583134766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745583134767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161467688424247, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7161467688424247}}
+:::MLLOG {"namespace": "", "time_ms": 1745583737025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745583746516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745583746516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71663555928639, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.71663555928639}}
+:::MLLOG {"namespace": "", "time_ms": 1745584340025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745584350995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745584350996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171961858159019, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7171961858159019}}
+:::MLLOG {"namespace": "", "time_ms": 1745584955665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745584965146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745584965146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7175660905383882, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7175660905383882}}
+:::MLLOG {"namespace": "", "time_ms": 1745585565937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745585575442, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745585575443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181211761065892, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7181211761065892}}
+:::MLLOG {"namespace": "", "time_ms": 1745586170179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745586179685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745586179685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7188269053186689, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7188269053186689}}
+:::MLLOG {"namespace": "", "time_ms": 1745586785370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745586794992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745586794993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191171118191311, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.7191171118191311}}
+:::MLLOG {"namespace": "", "time_ms": 1745587394486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745587403988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745587403988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7196645231474014, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7196645231474014}}
+:::MLLOG {"namespace": "", "time_ms": 1745587999303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3148992, "step_num": 32802}}
+:::MLLOG {"namespace": "", "time_ms": 1745588008978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3148992, "step_num": 32802, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745588008978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200902217910403, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3148992, "masked_lm_accuracy": 0.7200902217910403}}
+:::MLLOG {"namespace": "", "time_ms": 1745588008978, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3148992}}
+:::MLLOG {"namespace": "", "time_ms": 1745588008979, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_3.txt

@@ -0,0 +1,84 @@
+:::MLLOG {"namespace": "", "time_ms": 1745588028932, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745588028946, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745588028946, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745588028946, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745588028946, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745588029621, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745588029622, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745589719583, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745589729896, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745589729909, "event_type": "POINT_IN_TIME", "key": "seed", "value": 4120, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745143, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745144, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745145, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745146, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745146, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745589745146, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745589790515, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745590662059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745590701366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745590701367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3738187574204944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.3738187574204944}}
+:::MLLOG {"namespace": "", "time_ms": 1745591300740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745591310217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745591310217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.407555852049873, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.407555852049873}}
+:::MLLOG {"namespace": "", "time_ms": 1745591904870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745591914359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745591914359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020121426809402, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.5020121426809402}}
+:::MLLOG {"namespace": "", "time_ms": 1745592506867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745592517809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745592517809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6646601739383879, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6646601739383879}}
+:::MLLOG {"namespace": "", "time_ms": 1745593110284, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745593120055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745593120055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.703122741835458, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.703122741835458}}
+:::MLLOG {"namespace": "", "time_ms": 1745593713739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745593723210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745593723211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7083731492360433, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7083731492360433}}
+:::MLLOG {"namespace": "", "time_ms": 1745594317185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745594326671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745594326671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7107717264266241, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7107717264266241}}
+:::MLLOG {"namespace": "", "time_ms": 1745594919068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745594928715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745594928715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7129231032871065, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7129231032871065}}
+:::MLLOG {"namespace": "", "time_ms": 1745595522298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745595531920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745595531920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7140601788248334, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7140601788248334}}
+:::MLLOG {"namespace": "", "time_ms": 1745596125286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745596134779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745596134779, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7149945645105271, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7149945645105271}}
+:::MLLOG {"namespace": "", "time_ms": 1745596732267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745596741919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745596741920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7153055503254845, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7153055503254845}}
+:::MLLOG {"namespace": "", "time_ms": 1745597342010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745597351718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745597351718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7162052637054807, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7162052637054807}}
+:::MLLOG {"namespace": "", "time_ms": 1745597945037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745597954497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745597954497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7168858408927917, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7168858408927917}}
+:::MLLOG {"namespace": "", "time_ms": 1745598553278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745598562756, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745598562756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181580322129386, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7181580322129386}}
+:::MLLOG {"namespace": "", "time_ms": 1745599155685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745599165357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745599165357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7184280049233209, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7184280049233209}}
+:::MLLOG {"namespace": "", "time_ms": 1745599763495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745599773182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745599773182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7189048784119743, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7189048784119743}}
+:::MLLOG {"namespace": "", "time_ms": 1745600378094, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745600387579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745600387579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719234371752966, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.719234371752966}}
+:::MLLOG {"namespace": "", "time_ms": 1745600994209, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745601003674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745601003674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200033778236026, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7200033778236026}}
+:::MLLOG {"namespace": "", "time_ms": 1745601003674, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2699136}}
+:::MLLOG {"namespace": "", "time_ms": 1745601003675, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_4.txt

@@ -0,0 +1,90 @@
+:::MLLOG {"namespace": "", "time_ms": 1745601024651, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745601024665, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745601024665, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745601024665, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745601024665, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745601025341, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745601025341, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745602641535, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745602651782, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745602651795, "event_type": "POINT_IN_TIME", "key": "seed", "value": 31501, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667014, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667014, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667014, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667014, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667014, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667015, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745602667016, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745602720684, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745603593793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745603633838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745603633838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3756687879562378, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.3756687879562378}}
+:::MLLOG {"namespace": "", "time_ms": 1745604234034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745604243699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745604243700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.41726759388333273, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.41726759388333273}}
+:::MLLOG {"namespace": "", "time_ms": 1745604836741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745604846207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745604846207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6361151451156253, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.6361151451156253}}
+:::MLLOG {"namespace": "", "time_ms": 1745605440504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745605449980, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745605449981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7018973787625631, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.7018973787625631}}
+:::MLLOG {"namespace": "", "time_ms": 1745606043898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745606053400, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745606053400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7083071765445528, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.7083071765445528}}
+:::MLLOG {"namespace": "", "time_ms": 1745606645367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745606656313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745606656313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7105425567854018, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7105425567854018}}
+:::MLLOG {"namespace": "", "time_ms": 1745607248432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745607258080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745607258080, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7116558080627805, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7116558080627805}}
+:::MLLOG {"namespace": "", "time_ms": 1745607851555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745607861222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745607861223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7134888819285802, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7134888819285802}}
+:::MLLOG {"namespace": "", "time_ms": 1745608454820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745608464280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745608464280, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7140375443867275, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7140375443867275}}
+:::MLLOG {"namespace": "", "time_ms": 1745609055972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745609065428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745609065428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7148039085524422, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7148039085524422}}
+:::MLLOG {"namespace": "", "time_ms": 1745609658543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745609668034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745609668035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7156602161271232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7156602161271232}}
+:::MLLOG {"namespace": "", "time_ms": 1745610261103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745610270737, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745610270737, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.716008236294701, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.716008236294701}}
+:::MLLOG {"namespace": "", "time_ms": 1745610862227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745610871748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745610871748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7165316888264247, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7165316888264247}}
+:::MLLOG {"namespace": "", "time_ms": 1745611475921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745611485374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745611485374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171947257859367, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7171947257859367}}
+:::MLLOG {"namespace": "", "time_ms": 1745612083788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745612094912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745612094912, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181121451514108, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7181121451514108}}
+:::MLLOG {"namespace": "", "time_ms": 1745612686207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745612695807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745612695807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7187516025134495, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7187516025134495}}
+:::MLLOG {"namespace": "", "time_ms": 1745613288639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745613298492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745613298493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186400470279511, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7186400470279511}}
+:::MLLOG {"namespace": "", "time_ms": 1745613892142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745613901611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745613901611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7196106592814128, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7196106592814128}}
+:::MLLOG {"namespace": "", "time_ms": 1745614498093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745614507551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745614507551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719801956699008, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.719801956699008}}
+:::MLLOG {"namespace": "", "time_ms": 1745615109551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745615119233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745615119233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7204833462124779, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7204833462124779}}
+:::MLLOG {"namespace": "", "time_ms": 1745615119234, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2999040}}
+:::MLLOG {"namespace": "", "time_ms": 1745615119234, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_5.txt

@@ -0,0 +1,84 @@
+:::MLLOG {"namespace": "", "time_ms": 1745615138505, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745615138519, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745615138519, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745615138520, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745615138520, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745615139198, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745615139199, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745616751147, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745616761345, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745616761359, "event_type": "POINT_IN_TIME", "key": "seed", "value": 10057, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776690, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776691, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776692, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776693, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776693, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745616776693, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745616820038, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745617693199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745617733683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745617733683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37397653659184776, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37397653659184776}}
+:::MLLOG {"namespace": "", "time_ms": 1745618335201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745618344856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745618344856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3990937738191514, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.3990937738191514}}
+:::MLLOG {"namespace": "", "time_ms": 1745618939487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745618948957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745618948957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.46658501823743187, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.46658501823743187}}
+:::MLLOG {"namespace": "", "time_ms": 1745619545017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745619554509, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745619554509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6130311591284615, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6130311591284615}}
+:::MLLOG {"namespace": "", "time_ms": 1745620150097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745620159777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745620159777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.699844073113941, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.699844073113941}}
+:::MLLOG {"namespace": "", "time_ms": 1745620753981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745620764905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745620764906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7078652035622369, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7078652035622369}}
+:::MLLOG {"namespace": "", "time_ms": 1745621358745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745621368403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745621368404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7104341376395452, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7104341376395452}}
+:::MLLOG {"namespace": "", "time_ms": 1745621963465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745621973218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745621973218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7125999070349194, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7125999070349194}}
+:::MLLOG {"namespace": "", "time_ms": 1745622568888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745622578553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745622578553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7138977993102301, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7138977993102301}}
+:::MLLOG {"namespace": "", "time_ms": 1745623172627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745623182469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745623182469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147745728492737, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7147745728492737}}
+:::MLLOG {"namespace": "", "time_ms": 1745623788549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745623798248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745623798248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7157551918710982, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7157551918710982}}
+:::MLLOG {"namespace": "", "time_ms": 1745624393638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745624403132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745624403132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161134055682591, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7161134055682591}}
+:::MLLOG {"namespace": "", "time_ms": 1745625005439, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745625014940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745625014940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7176308717046466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7176308717046466}}
+:::MLLOG {"namespace": "", "time_ms": 1745625609773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745625619252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745625619252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7178728024164835, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7178728024164835}}
+:::MLLOG {"namespace": "", "time_ms": 1745626226550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745626236225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745626236226, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182088261558897, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7182088261558897}}
+:::MLLOG {"namespace": "", "time_ms": 1745626829609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745626839220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745626839221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7189385493596395, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7189385493596395}}
+:::MLLOG {"namespace": "", "time_ms": 1745627434203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745627443687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745627443688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7195644435428438, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7195644435428438}}
+:::MLLOG {"namespace": "", "time_ms": 1745628036848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745628047895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745628047895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7202766804468064, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7202766804468064}}
+:::MLLOG {"namespace": "", "time_ms": 1745628047896, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2699136}}
+:::MLLOG {"namespace": "", "time_ms": 1745628047896, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_6.txt

@@ -0,0 +1,90 @@
+:::MLLOG {"namespace": "", "time_ms": 1745628067289, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067303, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067303, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067303, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067303, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067966, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745628067966, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745629666209, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745629677049, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745629677063, "event_type": "POINT_IN_TIME", "key": "seed", "value": 12465, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691974, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691974, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691975, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745629691976, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745629740990, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745630621098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745630660651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745630660651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37397750247092476, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37397750247092476}}
+:::MLLOG {"namespace": "", "time_ms": 1745631266361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745631276039, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745631276039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4039459753604162, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4039459753604162}}
+:::MLLOG {"namespace": "", "time_ms": 1745631876011, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745631885494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745631885494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.46332124216215953, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.46332124216215953}}
+:::MLLOG {"namespace": "", "time_ms": 1745632483680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745632493164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745632493164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5909533494994754, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.5909533494994754}}
+:::MLLOG {"namespace": "", "time_ms": 1745633092727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745633102224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745633102224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.695507520153409, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.695507520153409}}
+:::MLLOG {"namespace": "", "time_ms": 1745633701690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745633711166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745633711166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.70634758018312, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.70634758018312}}
+:::MLLOG {"namespace": "", "time_ms": 1745634308886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745634318381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745634318382, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7093000508490063, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7093000508490063}}
+:::MLLOG {"namespace": "", "time_ms": 1745634917081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745634926554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745634926554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7113453558513096, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7113453558513096}}
+:::MLLOG {"namespace": "", "time_ms": 1745635523693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745635534738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745635534738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7130639910697937, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7130639910697937}}
+:::MLLOG {"namespace": "", "time_ms": 1745636131809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745636141329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745636141330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7139686294964381, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7139686294964381}}
+:::MLLOG {"namespace": "", "time_ms": 1745636739800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745636749545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745636749545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7148919718606132, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7148919718606132}}
+:::MLLOG {"namespace": "", "time_ms": 1745637359823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745637369305, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745637369306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7157319557099115, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7157319557099115}}
+:::MLLOG {"namespace": "", "time_ms": 1745637971557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745637981047, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745637981047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7163103591828119, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7163103591828119}}
+:::MLLOG {"namespace": "", "time_ms": 1745638579005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745638588492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745638588493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7172297625314622, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7172297625314622}}
+:::MLLOG {"namespace": "", "time_ms": 1745639192542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745639202337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745639202338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7177095975194658, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7177095975194658}}
+:::MLLOG {"namespace": "", "time_ms": 1745639798352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745639807842, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745639807842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181210790361677, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7181210790361677}}
+:::MLLOG {"namespace": "", "time_ms": 1745640406137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745640415674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745640415674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7187060095015027, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7187060095015027}}
+:::MLLOG {"namespace": "", "time_ms": 1745641012089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745641021561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745641021562, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191783354395912, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7191783354395912}}
+:::MLLOG {"namespace": "", "time_ms": 1745641631830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745641641317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745641641317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7195633859861464, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.7195633859861464}}
+:::MLLOG {"namespace": "", "time_ms": 1745642245529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745642255025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745642255026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7203016593342736, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7203016593342736}}
+:::MLLOG {"namespace": "", "time_ms": 1745642255026, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2999040}}
+:::MLLOG {"namespace": "", "time_ms": 1745642255026, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_7.txt

@@ -0,0 +1,102 @@
+:::MLLOG {"namespace": "", "time_ms": 1745642274895, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745642274909, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745642274909, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745642274909, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745642274909, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745642275592, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745642275592, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745643905216, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745643915672, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745643915685, "event_type": "POINT_IN_TIME", "key": "seed", "value": 11730, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930970, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930970, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930970, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930971, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745643930972, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745643991209, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745644864276, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745644905184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745644905184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3781838553292411, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.3781838553292411}}
+:::MLLOG {"namespace": "", "time_ms": 1745645504653, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745645514115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745645514116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4123739058063144, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4123739058063144}}
+:::MLLOG {"namespace": "", "time_ms": 1745646105727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745646115516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745646115516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4941163951442355, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.4941163951442355}}
+:::MLLOG {"namespace": "", "time_ms": 1745646708029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745646717701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745646717701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5998574114981152, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.5998574114981152}}
+:::MLLOG {"namespace": "", "time_ms": 1745647310461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745647319938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745647319938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6882066153344654, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.6882066153344654}}
+:::MLLOG {"namespace": "", "time_ms": 1745647910916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745647920440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745647920441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7018242018563406, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7018242018563406}}
+:::MLLOG {"namespace": "", "time_ms": 1745648513412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745648522931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745648522932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7060379794665745, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7060379794665745}}
+:::MLLOG {"namespace": "", "time_ms": 1745649115751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745649125216, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745649125217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7098249503544398, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7098249503544398}}
+:::MLLOG {"namespace": "", "time_ms": 1745649715710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745649725226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745649725226, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.710793800013406, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.710793800013406}}
+:::MLLOG {"namespace": "", "time_ms": 1745650317598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745650327086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745650327086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7119041085243225, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7119041085243225}}
+:::MLLOG {"namespace": "", "time_ms": 1745650919471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745650928949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745650928949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7127983939080011, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7127983939080011}}
+:::MLLOG {"namespace": "", "time_ms": 1745651519561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745651529185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745651529186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7138419349988302, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7138419349988302}}
+:::MLLOG {"namespace": "", "time_ms": 1745652122200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745652131782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745652131782, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147189242499216, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7147189242499216}}
+:::MLLOG {"namespace": "", "time_ms": 1745652730615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745652740312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745652740312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7152118898573376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7152118898573376}}
+:::MLLOG {"namespace": "", "time_ms": 1745653331444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745653340998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745653340998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.715751085962568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.715751085962568}}
+:::MLLOG {"namespace": "", "time_ms": 1745653933389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745653942898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745653942898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71591789268312, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.71591789268312}}
+:::MLLOG {"namespace": "", "time_ms": 1745654540794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745654550615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745654550616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.716970343816848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.716970343816848}}
+:::MLLOG {"namespace": "", "time_ms": 1745655142498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745655151972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745655151972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7177020226206098, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7177020226206098}}
+:::MLLOG {"namespace": "", "time_ms": 1745655742995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745655752627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745655752627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181210052399408, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.7181210052399408}}
+:::MLLOG {"namespace": "", "time_ms": 1745656351799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745656361268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745656361268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181035654885428, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7181035654885428}}
+:::MLLOG {"namespace": "", "time_ms": 1745656967639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3148992, "step_num": 32802}}
+:::MLLOG {"namespace": "", "time_ms": 1745656978577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3148992, "step_num": 32802, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745656978577, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7189842661221822, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3148992, "masked_lm_accuracy": 0.7189842661221822}}
+:::MLLOG {"namespace": "", "time_ms": 1745657575734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3298944, "step_num": 34364}}
+:::MLLOG {"namespace": "", "time_ms": 1745657585375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3298944, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3298944, "step_num": 34364, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745657585375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7194626552718026, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3298944, "masked_lm_accuracy": 0.7194626552718026}}
+:::MLLOG {"namespace": "", "time_ms": 1745658178120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3448896, "step_num": 35926}}
+:::MLLOG {"namespace": "", "time_ms": 1745658187683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3448896, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3448896, "step_num": 35926, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745658187683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7197951980999537, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3448896, "masked_lm_accuracy": 0.7197951980999537}}
+:::MLLOG {"namespace": "", "time_ms": 1745658780502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3598848, "step_num": 37488}}
+:::MLLOG {"namespace": "", "time_ms": 1745658789976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3598848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3598848, "step_num": 37488, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745658789976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200545878637404, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3598848, "masked_lm_accuracy": 0.7200545878637404}}
+:::MLLOG {"namespace": "", "time_ms": 1745658789977, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3598848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3598848}}
+:::MLLOG {"namespace": "", "time_ms": 1745658789977, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_8.txt

@@ -0,0 +1,84 @@
+:::MLLOG {"namespace": "", "time_ms": 1745658809591, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745658809604, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745658809604, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745658809604, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745658809604, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745658810272, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745658810273, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745660394086, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745660404397, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745660404410, "event_type": "POINT_IN_TIME", "key": "seed", "value": 30006, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419324, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419324, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419324, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419324, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419325, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745660419326, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745660468452, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745661348515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745661388786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745661388787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37456812262535094, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37456812262535094}}
+:::MLLOG {"namespace": "", "time_ms": 1745661996315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745662005821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745662005821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4560752814724332, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4560752814724332}}
+:::MLLOG {"namespace": "", "time_ms": 1745662609218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745662618735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745662618736, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6749432524045308, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.6749432524045308}}
+:::MLLOG {"namespace": "", "time_ms": 1745663219268, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745663228798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745663228798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7049068002473741, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.7049068002473741}}
+:::MLLOG {"namespace": "", "time_ms": 1745663830866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745663840454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745663840455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7094300275757199, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.7094300275757199}}
+:::MLLOG {"namespace": "", "time_ms": 1745664440883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745664450546, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745664450546, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7118306568690709, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7118306568690709}}
+:::MLLOG {"namespace": "", "time_ms": 1745665052620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745665062331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745665062331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7131405120804196, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7131405120804196}}
+:::MLLOG {"namespace": "", "time_ms": 1745665664579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745665674079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745665674079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7141566656884692, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7141566656884692}}
+:::MLLOG {"namespace": "", "time_ms": 1745666274721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745666285824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745666285824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.714853200458345, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.714853200458345}}
+:::MLLOG {"namespace": "", "time_ms": 1745666886316, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745666895839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745666895839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7154801947729929, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7154801947729929}}
+:::MLLOG {"namespace": "", "time_ms": 1745667505514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745667515034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745667515034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7160271604855856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7160271604855856}}
+:::MLLOG {"namespace": "", "time_ms": 1745668117399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745668126911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745668126912, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161095375106448, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7161095375106448}}
+:::MLLOG {"namespace": "", "time_ms": 1745668727427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745668736920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745668736921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7172707217080253, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7172707217080253}}
+:::MLLOG {"namespace": "", "time_ms": 1745669347896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745669357386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745669357386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7181751222837539, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7181751222837539}}
+:::MLLOG {"namespace": "", "time_ms": 1745669959126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745669968787, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745669968787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718704723176502, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.718704723176502}}
+:::MLLOG {"namespace": "", "time_ms": 1745670573606, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745670583206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745670583206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7194259950092861, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7194259950092861}}
+:::MLLOG {"namespace": "", "time_ms": 1745671185025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745671194623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745671194623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71981109891619, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.71981109891619}}
+:::MLLOG {"namespace": "", "time_ms": 1745671799743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745671810731, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745671810731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7204012291772025, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7204012291772025}}
+:::MLLOG {"namespace": "", "time_ms": 1745671810731, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2699136}}
+:::MLLOG {"namespace": "", "time_ms": 1745671810732, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/bert/result_9.txt

@@ -0,0 +1,90 @@
+:::MLLOG {"namespace": "", "time_ms": 1745671829193, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829206, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829207, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829207, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829207, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829888, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745671829889, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745673412514, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745673423084, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745673423098, "event_type": "POINT_IN_TIME", "key": "seed", "value": 27130, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438047, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438047, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438047, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438047, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438048, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745673438049, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745673490539, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745674363852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745674404525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745674404526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37596466257458644, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.37596466257458644}}
+:::MLLOG {"namespace": "", "time_ms": 1745675008663, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745675018130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745675018130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4192629811309633, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4192629811309633}}
+:::MLLOG {"namespace": "", "time_ms": 1745675616281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745675625972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745675625973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021054284913199, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.5021054284913199}}
+:::MLLOG {"namespace": "", "time_ms": 1745676222268, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745676232040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745676232041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6380213788577489, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.6380213788577489}}
+:::MLLOG {"namespace": "", "time_ms": 1745676829655, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745676839323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745676839324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7022705288160415, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.7022705288160415}}
+:::MLLOG {"namespace": "", "time_ms": 1745677435100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745677446056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745677446057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7073904724348159, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7073904724348159}}
+:::MLLOG {"namespace": "", "time_ms": 1745678041778, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745678051320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745678051320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7095591738110497, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7095591738110497}}
+:::MLLOG {"namespace": "", "time_ms": 1745678648680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745678658179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745678658180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7115420869418553, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7115420869418553}}
+:::MLLOG {"namespace": "", "time_ms": 1745679256309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745679265943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745679265943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7130387618428184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7130387618428184}}
+:::MLLOG {"namespace": "", "time_ms": 1745679861819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745679871292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745679871292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7135955617541359, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7135955617541359}}
+:::MLLOG {"namespace": "", "time_ms": 1745680468796, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745680478297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745680478297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7146279448554629, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.7146279448554629}}
+:::MLLOG {"namespace": "", "time_ms": 1745681075639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745681085107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745681085107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7151510505449205, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7151510505449205}}
+:::MLLOG {"namespace": "", "time_ms": 1745681680758, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745681690230, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745681690230, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7165615944635301, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7165615944635301}}
+:::MLLOG {"namespace": "", "time_ms": 1745682292031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745682301481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745682301482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7166609871955145, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7166609871955145}}
+:::MLLOG {"namespace": "", "time_ms": 1745682896874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745682907952, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745682907952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.717455704439254, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.717455704439254}}
+:::MLLOG {"namespace": "", "time_ms": 1745683523972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745683533688, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745683533689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7179811846642267, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7179811846642267}}
+:::MLLOG {"namespace": "", "time_ms": 1745684130394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745684139891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745684139891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185609034129552, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7185609034129552}}
+:::MLLOG {"namespace": "", "time_ms": 1745684736989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745684746439, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745684746439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191170454025269, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7191170454025269}}
+:::MLLOG {"namespace": "", "time_ms": 1745685345638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745685355274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745685355275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71960460628782, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.71960460628782}}
+:::MLLOG {"namespace": "", "time_ms": 1745685966333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745685976054, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745685976054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200535161154611, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7200535161154611}}
+:::MLLOG {"namespace": "", "time_ms": 1745685976055, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 2999040}}
+:::MLLOG {"namespace": "", "time_ms": 1745685976055, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_0.txt

@@ -0,0 +1,46 @@
+:::MLLOG {"namespace": "", "time_ms": 1745596628137, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}}
+:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}}
+:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 384}}
+:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}}
+:::MLLOG {"namespace": "", "time_ms": 1745596628178, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 387}}
+:::MLLOG {"namespace": "", "time_ms": 1745596629954, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 390}}
+:::MLLOG {"namespace": "", "time_ms": 1745596629955, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 391}}
+:::MLLOG {"namespace": "", "time_ms": 1745598065772, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 658}}
+:::MLLOG {"namespace": "", "time_ms": 1745598081470, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 394}}
+:::MLLOG {"namespace": "", "time_ms": 1745598081512, "event_type": "POINT_IN_TIME", "key": "seed", "value": 23282, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 395}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088273, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 507}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088274, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088275, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 514}}
+:::MLLOG {"namespace": "", "time_ms": 1745598088275, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 515}}
+:::MLLOG {"namespace": "", "time_ms": 1745598144406, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745605078062, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745605078063, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 1}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2608930553164607, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745610378469, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745616941326, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745616941327, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 2}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745622185857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.31207695716564665, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745622185858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745622185858, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745628713800, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745628713800, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 3}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745633828548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.32695300496649193, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745633828548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745633828549, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 527, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745640403678, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 598, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745640403679, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 603, "epoch_num": 4}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745645485614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.34190927146960864, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 679, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745645485615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 680, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745645485615, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 686, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_1.txt

@@ -0,0 +1,46 @@
+:::MLLOG {"namespace": "", "time_ms": 1745708052929, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}}
+:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}}
+:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}}
+:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}}
+:::MLLOG {"namespace": "", "time_ms": 1745708052970, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}}
+:::MLLOG {"namespace": "", "time_ms": 1745708055312, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}}
+:::MLLOG {"namespace": "", "time_ms": 1745708055312, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}}
+:::MLLOG {"namespace": "", "time_ms": 1745709484510, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}}
+:::MLLOG {"namespace": "", "time_ms": 1745709499880, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}}
+:::MLLOG {"namespace": "", "time_ms": 1745709499922, "event_type": "POINT_IN_TIME", "key": "seed", "value": 3218, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506804, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506805, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}}
+:::MLLOG {"namespace": "", "time_ms": 1745709506806, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}}
+:::MLLOG {"namespace": "", "time_ms": 1745709564057, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745716423332, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745716423333, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745721892086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2644758301871188, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745721892087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745721892087, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745728717917, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745728717918, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3183940553292647, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745734129092, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745740758848, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745740758849, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.330829179299047, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745746017219, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745752685505, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745752685506, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745757915230, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3430538198992862, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745757915231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745757915231, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_2.txt

@@ -0,0 +1,46 @@
+:::MLLOG {"namespace": "", "time_ms": 1745757942370, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}}
+:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}}
+:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}}
+:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}}
+:::MLLOG {"namespace": "", "time_ms": 1745757942411, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}}
+:::MLLOG {"namespace": "", "time_ms": 1745757943058, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}}
+:::MLLOG {"namespace": "", "time_ms": 1745757943059, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}}
+:::MLLOG {"namespace": "", "time_ms": 1745759379793, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}}
+:::MLLOG {"namespace": "", "time_ms": 1745759394363, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}}
+:::MLLOG {"namespace": "", "time_ms": 1745759394404, "event_type": "POINT_IN_TIME", "key": "seed", "value": 7068, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401265, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401265, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401266, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401267, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}}
+:::MLLOG {"namespace": "", "time_ms": 1745759401267, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}}
+:::MLLOG {"namespace": "", "time_ms": 1745759458864, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745766229351, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745766229352, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745771664180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.2618442233208197, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745771664180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745771664181, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745778271730, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745778271731, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745783649281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3050222595524408, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745783649281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745783649282, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745790269694, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745790269695, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745795622203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.33438554461867026, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745795622204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745795622204, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745802201592, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745802201593, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3413173788267323, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745807409644, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_3.txt

@@ -0,0 +1,46 @@
+:::MLLOG {"namespace": "", "time_ms": 1745708740629, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}}
+:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}}
+:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}}
+:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}}
+:::MLLOG {"namespace": "", "time_ms": 1745708740670, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}}
+:::MLLOG {"namespace": "", "time_ms": 1745708753515, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}}
+:::MLLOG {"namespace": "", "time_ms": 1745708753515, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}}
+:::MLLOG {"namespace": "", "time_ms": 1745710196875, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}}
+:::MLLOG {"namespace": "", "time_ms": 1745710211866, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}}
+:::MLLOG {"namespace": "", "time_ms": 1745710211906, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1934, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219928, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219929, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}}
+:::MLLOG {"namespace": "", "time_ms": 1745710219930, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}}
+:::MLLOG {"namespace": "", "time_ms": 1745710276595, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745717038732, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745717038733, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745722476155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.24994336549495808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745722476156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745722476156, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745729177485, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745729177486, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.30947442932060776, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745734589630, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745741107714, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745741107714, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3304143886715271, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745746523920, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745753134001, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745753134002, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745758428287, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3429861420134466, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745758428288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745758428288, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 684, "status": "success"}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_green/retinanet/result_4.txt

@@ -0,0 +1,45 @@
+:::MLLOG {"namespace": "", "time_ms": 1745758455763, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 380}}
+:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_green", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 381}}
+:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 382}}
+:::MLLOG {"namespace": "", "time_ms": 1745758455804, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 383}}
+:::MLLOG {"namespace": "", "time_ms": 1745758455805, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "retinanet", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 385}}
+:::MLLOG {"namespace": "", "time_ms": 1745758457940, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 388}}
+:::MLLOG {"namespace": "", "time_ms": 1745758457941, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 389}}
+:::MLLOG {"namespace": "", "time_ms": 1745759900517, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 656}}
+:::MLLOG {"namespace": "", "time_ms": 1745759915495, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 392}}
+:::MLLOG {"namespace": "", "time_ms": 1745759915536, "event_type": "POINT_IN_TIME", "key": "seed", "value": 25159, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 393}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922365, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 502}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 12191, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 503}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 259, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 504}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "epoch_count", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 505}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "first_epoch_num", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 506}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "adam", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 508}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922366, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 9.5e-05, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 509}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_weight_decay", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 510}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_epochs", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 511}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_factor", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 512}}
+:::MLLOG {"namespace": "", "time_ms": 1745759922367, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 513}}
+:::MLLOG {"namespace": "", "time_ms": 1745759981024, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745766937876, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745766937877, "event_type": "INTERVAL_START", "key": "eval_start", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 1}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.25660616888772175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 1}}
+:::MLLOG {"namespace": "", "time_ms": 1745772433927, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745779249804, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745779249805, "event_type": "INTERVAL_START", "key": "eval_start", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 2}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745784709047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3114751446994825, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745784709048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 2}}
+:::MLLOG {"namespace": "", "time_ms": 1745784709048, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745791366481, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745791366482, "event_type": "INTERVAL_START", "key": "eval_start", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 3}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.33395135022162803, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 3}}
+:::MLLOG {"namespace": "", "time_ms": 1745796796512, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 525, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745803562272, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 596, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745803562273, "event_type": "INTERVAL_START", "key": "eval_start", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 601, "epoch_num": 4}}
+
+:::MLLOG {"namespace": "", "time_ms": 1745808971898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3397162205764848, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 677, "epoch_num": 4}}
+:::MLLOG {"namespace": "", "time_ms": 1745808971899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 4, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 678, "epoch_num": 4}}

examples/mlperf/training_submission_v5.0/tinycorp/results/tinybox_red/bert/result_0.txt

@@ -0,0 +1,93 @@
+:::MLLOG {"namespace": "", "time_ms": 1745787472660, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "tinycorp", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 917}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472675, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "tinybox_red", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 918}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472675, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 919}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472675, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 920}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472675, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 922}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472820, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 925}}
+:::MLLOG {"namespace": "", "time_ms": 1745787472820, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 926}}
+:::MLLOG {"namespace": "", "time_ms": 1745789021684, "event_type": "POINT_IN_TIME", "key": "init_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1138}}
+:::MLLOG {"namespace": "", "time_ms": 1745789063002, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 929}}
+:::MLLOG {"namespace": "", "time_ms": 1745789063016, "event_type": "POINT_IN_TIME", "key": "seed", "value": 11341, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 930}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084054, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 96, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1007}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084054, "event_type": "POINT_IN_TIME", "key": "max_sequence_length", "value": 512, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1008}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084054, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1009}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084054, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "LAMB", "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1011}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.000175, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1012}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.01, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1013}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1014}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1015}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1016}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084055, "event_type": "POINT_IN_TIME", "key": "opt_lamb_epsilon", "value": 1e-06, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1017}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1019}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1020}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1021}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 37500, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1022}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1023}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10080, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1024}}
+:::MLLOG {"namespace": "", "time_ms": 1745789084056, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 3600000, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1025}}
+:::MLLOG {"namespace": "", "time_ms": 1745789130103, "event_type": "INTERVAL_START", "key": "epoch_start", "value": 0, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1055, "epoch_num": 0}}
+:::MLLOG {"namespace": "", "time_ms": 1745790243120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 149952, "step_num": 1562}}
+:::MLLOG {"namespace": "", "time_ms": 1745790285403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 149952, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 149952, "step_num": 1562, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745790285404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3725697540101551, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 149952, "masked_lm_accuracy": 0.3725697540101551}}
+:::MLLOG {"namespace": "", "time_ms": 1745791130486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 299904, "step_num": 3124}}
+:::MLLOG {"namespace": "", "time_ms": 1745791146651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 299904, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 299904, "step_num": 3124, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745791146651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4053410061768123, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 299904, "masked_lm_accuracy": 0.4053410061768123}}
+:::MLLOG {"namespace": "", "time_ms": 1745791993483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 449856, "step_num": 4686}}
+:::MLLOG {"namespace": "", "time_ms": 1745792009647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 449856, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 449856, "step_num": 4686, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745792009647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4546036266145252, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 449856, "masked_lm_accuracy": 0.4546036266145252}}
+:::MLLOG {"namespace": "", "time_ms": 1745792857040, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 599808, "step_num": 6248}}
+:::MLLOG {"namespace": "", "time_ms": 1745792873247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 599808, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 599808, "step_num": 6248, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745792873248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5833114096096583, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 599808, "masked_lm_accuracy": 0.5833114096096583}}
+:::MLLOG {"namespace": "", "time_ms": 1745793720402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 749760, "step_num": 7810}}
+:::MLLOG {"namespace": "", "time_ms": 1745793736590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 749760, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 749760, "step_num": 7810, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745793736590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6891417679332551, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 749760, "masked_lm_accuracy": 0.6891417679332551}}
+:::MLLOG {"namespace": "", "time_ms": 1745794586055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 899712, "step_num": 9372}}
+:::MLLOG {"namespace": "", "time_ms": 1745794602267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 899712, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 899712, "step_num": 9372, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745794602267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7027380545934041, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 899712, "masked_lm_accuracy": 0.7027380545934041}}
+:::MLLOG {"namespace": "", "time_ms": 1745795451897, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1049664, "step_num": 10934}}
+:::MLLOG {"namespace": "", "time_ms": 1745795468074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1049664, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1049664, "step_num": 10934, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745795468074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7082311675662086, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1049664, "masked_lm_accuracy": 0.7082311675662086}}
+:::MLLOG {"namespace": "", "time_ms": 1745796316509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1199616, "step_num": 12496}}
+:::MLLOG {"namespace": "", "time_ms": 1745796332716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1199616, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1199616, "step_num": 12496, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745796332716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7108195407049996, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1199616, "masked_lm_accuracy": 0.7108195407049996}}
+:::MLLOG {"namespace": "", "time_ms": 1745797182147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1349568, "step_num": 14058}}
+:::MLLOG {"namespace": "", "time_ms": 1745797198362, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1349568, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1349568, "step_num": 14058, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745797198363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7119770498502822, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1349568, "masked_lm_accuracy": 0.7119770498502822}}
+:::MLLOG {"namespace": "", "time_ms": 1745798047629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1499520, "step_num": 15620}}
+:::MLLOG {"namespace": "", "time_ms": 1745798063852, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1499520, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1499520, "step_num": 15620, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745798063852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7129385369164604, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1499520, "masked_lm_accuracy": 0.7129385369164604}}
+:::MLLOG {"namespace": "", "time_ms": 1745798917116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1649472, "step_num": 17182}}
+:::MLLOG {"namespace": "", "time_ms": 1745798933320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1649472, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1649472, "step_num": 17182, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745798933320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.713829753512428, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1649472, "masked_lm_accuracy": 0.713829753512428}}
+:::MLLOG {"namespace": "", "time_ms": 1745799781286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1799424, "step_num": 18744}}
+:::MLLOG {"namespace": "", "time_ms": 1745799797515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1799424, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1799424, "step_num": 18744, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745799797515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7143420588402521, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1799424, "masked_lm_accuracy": 0.7143420588402521}}
+:::MLLOG {"namespace": "", "time_ms": 1745800650547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 1949376, "step_num": 20306}}
+:::MLLOG {"namespace": "", "time_ms": 1745800666777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 1949376, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 1949376, "step_num": 20306, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745800666777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7151911968276614, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 1949376, "masked_lm_accuracy": 0.7151911968276614}}
+:::MLLOG {"namespace": "", "time_ms": 1745801513018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2099328, "step_num": 21868}}
+:::MLLOG {"namespace": "", "time_ms": 1745801529219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2099328, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2099328, "step_num": 21868, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745801529219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7162183267729623, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2099328, "masked_lm_accuracy": 0.7162183267729623}}
+:::MLLOG {"namespace": "", "time_ms": 1745802383777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2249280, "step_num": 23430}}
+:::MLLOG {"namespace": "", "time_ms": 1745802399993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2249280, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2249280, "step_num": 23430, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745802399993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7163212441262745, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2249280, "masked_lm_accuracy": 0.7163212441262745}}
+:::MLLOG {"namespace": "", "time_ms": 1745803252264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2399232, "step_num": 24992}}
+:::MLLOG {"namespace": "", "time_ms": 1745803268480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2399232, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2399232, "step_num": 24992, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745803268480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7175917211033049, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2399232, "masked_lm_accuracy": 0.7175917211033049}}
+:::MLLOG {"namespace": "", "time_ms": 1745804120412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2549184, "step_num": 26554}}
+:::MLLOG {"namespace": "", "time_ms": 1745804136630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2549184, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2549184, "step_num": 26554, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745804136630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7178130229314168, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2549184, "masked_lm_accuracy": 0.7178130229314168}}
+:::MLLOG {"namespace": "", "time_ms": 1745804990109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2699136, "step_num": 28116}}
+:::MLLOG {"namespace": "", "time_ms": 1745805006323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2699136, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2699136, "step_num": 28116, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745805006324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182639241218567, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2699136, "masked_lm_accuracy": 0.7182639241218567}}
+:::MLLOG {"namespace": "", "time_ms": 1745805853485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2849088, "step_num": 29678}}
+:::MLLOG {"namespace": "", "time_ms": 1745805869680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2849088, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2849088, "step_num": 29678, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745805869680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185757029624212, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2849088, "masked_lm_accuracy": 0.7185757029624212}}
+:::MLLOG {"namespace": "", "time_ms": 1745806722533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 2999040, "step_num": 31240}}
+:::MLLOG {"namespace": "", "time_ms": 1745806738689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 2999040, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 2999040, "step_num": 31240, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745806738690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7192848898115612, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 2999040, "masked_lm_accuracy": 0.7192848898115612}}
+:::MLLOG {"namespace": "", "time_ms": 1745807591120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1106, "epoch_num": 3148992, "step_num": 32802}}
+:::MLLOG {"namespace": "", "time_ms": 1745807607349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1159, "epoch_count": 3148992, "step_num": 32802, "samples_count": 10080}}
+:::MLLOG {"namespace": "", "time_ms": 1745807607349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200041702815465, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1160, "epoch_num": 3148992, "masked_lm_accuracy": 0.7200041702815465}}
+:::MLLOG {"namespace": "", "time_ms": 1745807607350, "event_type": "POINT_IN_TIME", "key": "epoch_stop", "value": 3148992, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1178, "epoch_num": 3148992}}
+:::MLLOG {"namespace": "", "time_ms": 1745807607350, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "tinygrad/examples/mlperf/model_train.py", "lineno": 1179, "status": "success"}}