From 0820c7a4248cb2d062f418b458e58701a7ee7e50 Mon Sep 17 00:00:00 2001 From: Christopher Milan Date: Wed, 26 Aug 2026 15:53:55 -0700 Subject: [PATCH] benchmarks: openpilot matrix (#17714) --- .github/workflows/benchmark.yml | 90 ++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 798d2261b3..ca506e4d95 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -446,14 +446,55 @@ jobs: - name: UsbGPU (USB4/TB) tiny tests run: DEV=PCI+NV:NAK python3.11 test/test_tiny.py - testcommalatest: - name: comma Benchmark (0.11.2) + testcomma: + strategy: + matrix: + dev: ['QCOM', 'QCOM:IR3'] + version: ['0.11.0', '0.11.2'] + model: ['vision', 'policy', 'supercombo', 'dmonitoring'] + # exclude non-existent models + exclude: [{ version: '0.11.0', model: supercombo }, { version: '0.11.2', model: vision }, { version: '0.11.2', model: policy }] + include: + - version: '0.11.0' + model: vision + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx + timing: 17 + - version: '0.11.0' + model: policy + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx + timing: 3.2 + - version: '0.11.0' + model: dmonitoring + url: https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + timing: 11 + - version: '0.11.2' + model: supercombo + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b + timing: 26 + - dev: QCOM:IR3 + version: '0.11.2' + model: supercombo + timing: 41 + - version: '0.11.2' + model: dmonitoring + url: https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 + timing: 11 + # IR3 dmonitoring is slightly slower + - dev: QCOM:IR3 + model: dmonitoring + timing: 12 + fail-fast: false + name: openpilot ${{ matrix.version }} compile3 ${{ matrix.model }} (DEV=${{ matrix.dev }}) runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 + timeout-minutes: 5 defaults: run: shell: bash -e -o pipefail {0} if: github.repository_owner == 'tinygrad' + env: + DEV: ${{ matrix.dev }} + ASSERT_MIN_STEP_TIME: ${{ matrix.timing }} + BENCHMARK_LOG: ${{ matrix.dev == 'QCOM:IR3' && 'ir3_' || '' }}openpilot_${{ matrix.version }}_${{ matrix.model }} steps: - name: Checkout Code uses: actions/checkout@v6 @@ -464,45 +505,10 @@ jobs: rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: reset process replay run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 supercombo (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_2_supercombo_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.2 supercombo - run: BENCHMARK_LOG=ir3_openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=41 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b - - name: openpilot compile3 0.11.2 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_2_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316 - - name: Run process replay tests - uses: ./.github/actions/process-replay - - testcommaold: - name: comma Benchmark (0.11.0) - runs-on: [self-hosted, Linux, comma] - timeout-minutes: 12 - defaults: - run: - shell: bash -e -o pipefail {0} - if: github.repository_owner == 'tinygrad' - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - name: setup staging db - if: github.ref == 'refs/heads/update_benchmark_staging' - run: | - echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV - rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - - name: reset process replay - run: test/external/process_replay/reset.py - - name: openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_vision (from pickle) - run: BENCHMARK_LOG=openpilot_0_11_0_vision_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py - - name: IR3 openpilot compile3 0.11.0 driving_vision - run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=18 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx - - name: openpilot compile3 0.11.0 driving_policy - run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx - - name: openpilot compile3 0.11.0 dmonitoring - run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx + - name: compile + run: FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py ${{ matrix.url }} + - name: run pickle + run: BENCHMARK_LOG="${BENCHMARK_LOG}_run_pickle" RUN_PICKLE=1 taskset -c 4-7 python3 examples/openpilot/compile3.py - name: Run process replay tests uses: ./.github/actions/process-replay