forked from tinygrad/tinygrad
614 lines
28 KiB
YAML
614 lines
28 KiB
YAML
name: Benchmarks
|
|
env:
|
|
# TODO: this rescheduling makes gpt2, mixtral and llama unjitted slower
|
|
# TODO: very slow for llama 70B and resnet training 6 GPU
|
|
CAPTURE_PROCESS_REPLAY: "1"
|
|
ASSERT_PROCESS_REPLAY: "0"
|
|
PYTHONPATH: .
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- update_benchmark
|
|
- update_benchmark_staging
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
# the goal of this test is to replicate a normal person on a laptop running the test
|
|
# no process replay, no benchmarks, no CI, just a normal laptop person
|
|
# the 3 minute timeout should not be raised
|
|
testmacpytest:
|
|
name: Mac pytest
|
|
env:
|
|
CI: ""
|
|
CAPTURE_PROCESS_REPLAY: "0"
|
|
runs-on: [self-hosted, macOS]
|
|
timeout-minutes: 4
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
# brew install uv
|
|
- name: setup python environment
|
|
run: |
|
|
rm -rf /tmp/tinygrad_pytest_ci
|
|
uv venv /tmp/tinygrad_pytest_ci
|
|
source /tmp/tinygrad_pytest_ci/bin/activate
|
|
uv pip install .[testing]
|
|
- name: setup staging db
|
|
run: |
|
|
echo "CACHEDB=/tmp/pytest-db-ci.db" >> $GITHUB_ENV
|
|
rm -f /tmp/pytest-db-ci*
|
|
- name: Run pytest -nauto
|
|
run: |
|
|
source /tmp/tinygrad_pytest_ci/bin/activate
|
|
pytest -nauto --durations=20
|
|
- name: openpilot compile3 0.10.1 driving_vision
|
|
run: FLOAT16=1 DEV=CL IMAGE=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx
|
|
|
|
# TODO: reenable when not flaky
|
|
#testframeworkpytest:
|
|
# name: framework pytest
|
|
# env:
|
|
# CI: ""
|
|
# CAPTURE_PROCESS_REPLAY: "0"
|
|
# runs-on: [self-hosted, framework]
|
|
# timeout-minutes: 10
|
|
# defaults:
|
|
# run:
|
|
# shell: bash -e -o pipefail {0}
|
|
# if: github.repository_owner == 'tinygrad'
|
|
# steps:
|
|
# - name: Checkout Code
|
|
# uses: actions/checkout@v6
|
|
# - name: setup python environment
|
|
# run: |
|
|
# rm -rf /tmp/tinygrad_pytest_ci
|
|
# uv venv /tmp/tinygrad_pytest_ci
|
|
# source /tmp/tinygrad_pytest_ci/bin/activate
|
|
# uv pip install .[testing]
|
|
# - name: setup staging db
|
|
# run: |
|
|
# echo "CACHEDB=/tmp/pytest-db-ci.db" >> $GITHUB_ENV
|
|
# rm -f /tmp/pytest-db-ci*
|
|
# - name: Run pytest -nauto
|
|
# run: |
|
|
# source /tmp/tinygrad_pytest_ci/bin/activate
|
|
# pytest -nauto --durations=20
|
|
|
|
llmbenchmark:
|
|
name: LLM (DEV=${{ matrix.dev }})
|
|
runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['METAL', 'AMD', 'NV']
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup (AMD)
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py amd rmmod
|
|
./extra/hcq/hcq_smi.py amd kill_pids
|
|
- name: Symlink models and datasets
|
|
run: |
|
|
mkdir -p weights
|
|
ln -s /raid/weights/LLaMA-3 weights/LLaMA-3
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: python3 test/external/process_replay/reset.py
|
|
- name: Run llama3.2
|
|
run: BENCHMARK_LOG=llama32_3b-f16 JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m llama3.2:3b-f16 --benchmark --warmup
|
|
- name: Run qwen3.6
|
|
# qwen3.6:35b-a3b doesn't fit on mac
|
|
if: ${{ matrix.dev != 'METAL' }}
|
|
run: BENCHMARK_LOG=qwen36_35b-a3b JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m qwen3.6:35b-a3b --benchmark --warmup
|
|
- name: Run olmoe
|
|
# just metal for now
|
|
if: ${{ matrix.dev == 'METAL' }}
|
|
run: BENCHMARK_LOG=olmoe JITBEAM=2 IGNORE_BEAM_CACHE=1 python3 -m tinygrad.llm -m olmoe --benchmark --warmup
|
|
- name: Run LLaMA-3 8B on 4 GPUs with BEAM
|
|
# only run on machines with multiple gpus
|
|
if: ${{ matrix.dev != 'METAL' }}
|
|
run: BENCHMARK_LOG=llama3_beam_4gpu JITBEAM=2 IGNORE_BEAM_CACHE=1 CAPTURE_PROCESS_REPLAY=0 python3 examples/llama3.py --size 8B --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark --temperature 0
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
cifarbenchmark:
|
|
name: HLB-CIFAR10 (DEV=${{ matrix.dev }})
|
|
runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['METAL', 'AMD', 'NV']
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup (AMD)
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py amd rmmod
|
|
./extra/hcq/hcq_smi.py amd kill_pids
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: python3 test/external/process_replay/reset.py
|
|
- name: Run 10 CIFAR training steps
|
|
env:
|
|
ASSERT_MIN_STEP_TIME: ${{ matrix.dev == 'NV' && '130' || matrix.dev == 'AMD' && '200' || '3000' }}
|
|
run: BENCHMARK_LOG=cifar_10steps STEPS=10 python3 examples/hlb_cifar10.py
|
|
- name: Run 10 CIFAR training steps w HALF
|
|
env:
|
|
ASSERT_MIN_STEP_TIME: ${{ matrix.dev == 'NV' && '120' || matrix.dev == 'AMD' && '235' || '3000' }}
|
|
run: BENCHMARK_LOG=cifar_10steps_half STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py
|
|
- name: Run full CIFAR training w 1 GPU
|
|
# slow on metal
|
|
if: ${{ matrix.dev != 'METAL' }}
|
|
run: time BENCHMARK_LOG=cifar DEFAULT_FLOAT=HALF STEPS=1000 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py
|
|
- name: Run full CIFAR training steps w 6 GPUS
|
|
# only run on machines with multiple gpus
|
|
if: ${{ matrix.dev != 'METAL' }}
|
|
run: time BENCHMARK_LOG=cifar_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.0 python3 examples/hlb_cifar10.py
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
mlperfbenchmark:
|
|
name: MLPerf (${{ matrix.dev }})
|
|
runs-on: [self-hosted, Linux, "${{ matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['AMD', 'NV']
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup (AMD)
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py amd rmmod
|
|
./extra/hcq/hcq_smi.py amd kill_pids
|
|
- name: Symlink models and datasets
|
|
run: |
|
|
mkdir -p extra/datasets
|
|
ln -s /raid/datasets/imagenet extra/datasets/imagenet
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: Run MLPerf resnet eval on training data
|
|
run: time BENCHMARK_LOG=resnet_eval MODEL=resnet python3 examples/mlperf/model_eval.py
|
|
- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
|
|
run: BENCHMARK_LOG=resnet_10steps DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py
|
|
- name: Run 10 MLPerf ResNet50 training steps (6 gpu)
|
|
run: BENCHMARK_LOG=resnet_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py
|
|
- name: Run 10 MLPerf Bert training steps (6 gpu)
|
|
# TODO: remove BERT_LAYERS once scheduler is fast
|
|
run: BENCHMARK_LOG=bert_10steps_6gpu CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=72 GPUS=6 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
sdbenchmark:
|
|
name: Stable Diffusion (DEV=${{ matrix.dev }})
|
|
runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['METAL', 'AMD', 'NV']
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup (AMD)
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py amd rmmod
|
|
./extra/hcq/hcq_smi.py amd kill_pids
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: python3 test/external/process_replay/reset.py
|
|
- name: Run Stable Diffusion
|
|
env:
|
|
ASSERT_MIN_STEP_TIME: ${{ matrix.dev == 'METAL' && '720' || matrix.dev == 'AMD' && '550' || '0' }}
|
|
run: BENCHMARK_LOG=stable_diffusion python3 examples/stable_diffusion.py --fp16 --seed 0 --noshow --timing
|
|
- name: Run SDXL
|
|
if: ${{ matrix.dev != 'NV' }}
|
|
env:
|
|
ASSERT_MIN_STEP_TIME: ${{ matrix.dev == 'METAL' && '5000' || matrix.dev == 'AMD' && '3200' || '2000' }}
|
|
run: BENCHMARK_LOG=stable_diffusion_xl CAPTURE_PROCESS_REPLAY=0 python3 examples/sdxl.py --seed 0 --noshow --timing
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
tests:
|
|
name: Tests (DEV=${{ matrix.dev }})
|
|
runs-on: [self-hosted, "${{ matrix.dev == 'METAL' && 'macOS' || matrix.dev == 'AMD' && 'tinybox' || 'tinyboxgreen' }}"]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['METAL', 'AMD', 'NV']
|
|
timeout-minutes: 60
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup (AMD)
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py amd rmmod
|
|
./extra/hcq/hcq_smi.py amd kill_pids
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: python3 test/external/process_replay/reset.py
|
|
- name: Test tiny
|
|
run: |
|
|
DEBUG=2 python -m pytest -rA test/test_tiny.py
|
|
if [[ "${{ matrix.dev }}" == "NV" ]]; then
|
|
DEBUG=2 DEV=CUDA python -m pytest -rA test/test_tiny.py
|
|
fi
|
|
- name: Test tensor cores
|
|
run: |
|
|
if [[ "${{ matrix.dev }}" == "METAL" ]]; then
|
|
python3 test/opt/test_tensor_cores.py
|
|
DEBUG=2 SHOULD_USE_TC=1 python3 extra/gemm/simple_matmul.py
|
|
DEBUG=2 SHOULD_USE_TC=1 HALF=1 python3 extra/gemm/simple_matmul.py
|
|
DEBUG=2 SHOULD_USE_TC=1 BFLOAT16=1 python3 extra/gemm/simple_matmul.py
|
|
M_START=6 M_STOP=10 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=6 K_STOP=24 K_STEP=1 TC_OPT=2 DEBUG=2 python3 ./extra/gemm/fuzz_matmul.py
|
|
elif [[ "${{ matrix.dev }}" == "NV" ]]; then
|
|
ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
|
|
DEV=NV:PTX ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
|
|
DEV=CUDA SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
DEV=CUDA SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
DEV=CUDA SHOULD_USE_TC=1 ALLOW_TF32=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py
|
|
DEV=CUDA SHOULD_USE_TC=1 FP8E4M3=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
DEV=NV:PTX SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
SHOULD_USE_TC=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
# TODO: too slow
|
|
# M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
|
|
# DEV=NV:PTX M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
|
|
else
|
|
python3 test/opt/test_tensor_cores.py
|
|
# TODO: this is flaky
|
|
# DEV=AMD:LLVM python3 test/opt/test_tensor_cores.py
|
|
SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py
|
|
# TODO: AMD compiler bug causes this to fail
|
|
# HSA=1 M_START=12 M_STOP=20 M_STEP=1 N_START=12 N_STOP=20 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 DEBUG=2 python3 ./extra/gemm/fuzz_matmul.py
|
|
fi
|
|
- name: Run model inference benchmark
|
|
# TODO: unstable on AMD
|
|
if: ${{ matrix.dev != 'AMD' }}
|
|
run: CAPTURE_PROCESS_REPLAY=0 NOCLANG=1 python3 test/external/external_model_benchmark.py
|
|
- name: Test speed vs torch
|
|
# TODO: unstable on AMD
|
|
if: ${{ matrix.dev != 'AMD' }}
|
|
env:
|
|
HALF: ${{ matrix.dev == 'NV' && '1' || '0' }}
|
|
run: CAPTURE_PROCESS_REPLAY=0 BIG=2 ${{ matrix.dev == 'METAL' && 'MPS=1' || 'TORCHCUDA=1' }} python3 test/speed/external_test_speed_v_torch.py
|
|
- name: Test speed vs theoretical
|
|
# no targets for METAL
|
|
if: ${{ matrix.dev != 'METAL' }}
|
|
run: IGNORE_BEAM_CACHE=1 CCACHE=0 BEAM_DEBUG=1 DEBUG=1 python -m pytest -rA test/external/speed_v_theoretical.py --durations=20
|
|
- name: Train MNIST
|
|
run: time TARGET_EVAL_ACC_PCT=96.0 python3 examples/beautiful_mnist.py
|
|
- name: Test benchmark allreduce
|
|
if: ${{ matrix.dev == 'NV' }}
|
|
run: python test/external/external_benchmark_multitensor_allreduce.py
|
|
- name: HEVC Decode Benchmark
|
|
if: ${{ matrix.dev == 'NV' }}
|
|
run: VALIDATE=1 MAX_FRAMES=100 ASSERT_FPS=1400 JITBEAM=1 PYTHONPATH=. python3 extra/hevc/decode.py
|
|
- uses: actions/upload-artifact@v7
|
|
if: ${{ matrix.dev != 'AMD' }}
|
|
with:
|
|
name: Speed (${{ matrix.dev }})
|
|
path: |
|
|
onnx_inference_speed.csv
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
testusbgpu:
|
|
name: UsbGPU Benchmark
|
|
runs-on: [self-hosted, macOS]
|
|
timeout-minutes: 10
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: Kill stale pids
|
|
run: |
|
|
PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
|
|
PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids
|
|
# since sudo is required for usbgpu on macos, do not write bytecode, as some of the files are owned by root
|
|
- name: UsbGPU boot time
|
|
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus
|
|
- name: UsbGPU tiny tests
|
|
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py
|
|
- name: UsbGPU copy speeds
|
|
run: sudo -E PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
|
#- name: UsbGPU openpilot test
|
|
# run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx
|
|
- name: UsbGPU (USB4/TB) install script
|
|
run: PYTHONPATH=. sh extra/setup_tinygpu_osx.sh
|
|
- name: UsbGPU (USB4/TB) boot time
|
|
run: PYTHONPATH=. DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus
|
|
- name: UsbGPU (USB4/TB) tiny tests
|
|
run: PYTHONPATH=. DEV=PCI+NV:NAK python3.11 test/test_tiny.py
|
|
|
|
testcommalatest:
|
|
name: comma Benchmark (0.11.2)
|
|
runs-on: [self-hosted, Linux, comma]
|
|
timeout-minutes: 12
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: openpilot compile3 0.11.2 supercombo
|
|
run: BENCHMARK_LOG=openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b
|
|
- name: openpilot compile3 0.11.2 supercombo (from pickle)
|
|
run: BENCHMARK_LOG=openpilot_0_11_2_supercombo_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=26 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py
|
|
- name: IR3 openpilot compile3 0.11.2 supercombo
|
|
run: BENCHMARK_LOG=ir3_openpilot_0_11_2_supercombo PYTHONPATH="." ASSERT_MIN_STEP_TIME=41 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/433f85f956837606ad1f1cbee4aa7e2158ad23c768dea914b20436c97232741b
|
|
- name: openpilot compile3 0.11.2 dmonitoring
|
|
run: BENCHMARK_LOG=openpilot_0_11_2_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/3e7b31dfbc0a5234f1baf196513b77fc6af12204b8a8ffe8ee0417e48352f316
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
testcommaold:
|
|
name: comma Benchmark (0.11.0)
|
|
runs-on: [self-hosted, Linux, comma]
|
|
timeout-minutes: 12
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: openpilot compile3 0.11.0 driving_vision
|
|
run: BENCHMARK_LOG=openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
|
|
- name: openpilot compile3 0.11.0 driving_vision (from pickle)
|
|
run: BENCHMARK_LOG=openpilot_0_11_0_vision_run_pickle RUN_PICKLE=1 PYTHONPATH="." ASSERT_MIN_STEP_TIME=17 DEV=QCOM taskset -c 4-7 python3 examples/openpilot/compile3.py
|
|
- name: IR3 openpilot compile3 0.11.0 driving_vision
|
|
run: BENCHMARK_LOG=ir3_openpilot_0_11_0_vision PYTHONPATH="." ASSERT_MIN_STEP_TIME=18 DEV=QCOM:IR3 FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_vision.onnx
|
|
- name: openpilot compile3 0.11.0 driving_policy
|
|
run: BENCHMARK_LOG=openpilot_0_11_0_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3.2 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/driving_policy.onnx
|
|
- name: openpilot compile3 0.11.0 dmonitoring
|
|
run: BENCHMARK_LOG=openpilot_0_11_0_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=1 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/v0.11.0/selfdrive/modeld/models/dmonitoring_model.onnx
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
testqualcommdsp:
|
|
name: DSP Benchmark
|
|
runs-on: [self-hosted, Linux, comma4]
|
|
timeout-minutes: 5
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: benchmark MobileNetV2 on DSP
|
|
run: |
|
|
# generate quantized weights
|
|
ln -s /data/home/tiny/tinygrad/extra/datasets/imagenet extra/datasets/imagenet
|
|
ln -s /data/home/tiny/tinygrad/testsig-*.so .
|
|
PYTHONPATH=. DEV=CPU QUANT=1 CNT=0 python3 examples/test_onnx_imagenet.py https://github.com/xamcat/mobcat-samples/raw/refs/heads/master/onnx_runtime/InferencingSample/InferencingSample/mobilenetv2-7.onnx /tmp/model.quant.onnx
|
|
# benchmark on DSP with NOOPT=1, the devectorizer has issues
|
|
PYTHONPATH=. DEV=DSP NOOPT=1 CNT=2 DEBUG=2 python3 examples/test_onnx_imagenet.py /tmp/model.quant.onnx
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
testcommausbgpubenchmark:
|
|
name: UsbGPU Benchmark (comma)
|
|
runs-on: [self-hosted, Linux, comma4]
|
|
timeout-minutes: 20
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: openpilot compile3 big_driving_supercombo
|
|
run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo PICKLE_OOB=1 PYTHONPATH="." TC_OPT=2 GMMU=0 DEV=USB+AMD:LLVM ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py https://gitlab.com/commaai/openpilot-lfs.git/gitlab-lfs/objects/10926f2c0911821ca0e72439c1c3bf3ec11f0a08789aa14b7ee8f25379b2afa4 openpilot.pkl
|
|
- name: openpilot load_pickle big_driving_supercombo
|
|
run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_load_pickle PICKLE_OOB=1 PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_LOAD_TIME=25 python3 examples/openpilot/load_pickle.py openpilot.pkl
|
|
- name: openpilot run_pickle big_driving_supercombo
|
|
run: BENCHMARK_LOG=usbgpu_openpilot_big_driving_supercombo_run_pickle RUN_PICKLE=1 PICKLE_OOB=1 PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py - openpilot.pkl
|
|
- name: Test copy speeds
|
|
run: SIZE=64e6 PYTHONPATH=. GMMU=0 DEV=USB+AMD python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds
|
|
|
|
driverbenchmarks:
|
|
name: PCI Driver Benchmark (DEV=${{ matrix.dev }})
|
|
runs-on: [self-hosted, Linux, tinyboxrandom]
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
dev: ['AMD', 'NV']
|
|
timeout-minutes: 20
|
|
defaults:
|
|
run:
|
|
shell: bash -e -o pipefail {0}
|
|
env:
|
|
DEV: ${{ matrix.dev }}
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Setup
|
|
run: |
|
|
./extra/amdpci/setup_python_cap.sh
|
|
./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} rmmod
|
|
./extra/hcq/hcq_smi.py ${{ matrix.dev == 'AMD' && 'amd' || 'nv' }} kill_pids
|
|
mkdir -p extra/datasets
|
|
ln -s /raid/datasets/imagenet extra/datasets/imagenet
|
|
- name: setup staging db
|
|
if: github.ref == 'refs/heads/update_benchmark_staging'
|
|
run: |
|
|
echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
|
|
rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
|
|
- name: reset process replay
|
|
run: test/external/process_replay/reset.py
|
|
- name: Test driver cold start time
|
|
run: time DEBUG=3 AM_RESET=1 python3 test/test_tiny.py TestTiny.test_plus
|
|
- name: Test driver warm start time
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: time DEBUG=3 python3 test/test_tiny.py TestTiny.test_plus
|
|
- name: Test GPU crash recovery
|
|
if: ${{ matrix.dev == 'AMD' }}
|
|
run: python3 -m pytest -rA test/external/external_test_gpu_crash.py
|
|
- name: Test tensor cores
|
|
run: |
|
|
if [[ "${{ matrix.dev }}" == "AMD" ]]; then
|
|
# Fails on 9070
|
|
# python3 test/test_linearizer.py test/opt/test_tensor_cores.py
|
|
# DEV=AMD:LLVM python3 test/test_linearizer.py test/opt/test_tensor_cores.py
|
|
# SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
|
|
SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py
|
|
else
|
|
ALLOW_TF32=1 python3 test/opt/test_tensor_cores.py
|
|
fi
|
|
- name: Test DISK copy time
|
|
run: TESTFILE=/raid/downloads/llama3-8b-sfr/model-00001-of-00004.safetensors python3 test/external/external_benchmark_disk_raw.py
|
|
- name: Test CPU copy time
|
|
run: |
|
|
GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyDefaulttoCPUJit
|
|
GRAPH_ONE_KERNEL=1 NSZ=8192 python3 test/speed/external_test_copy_speed.py TestCopySpeed.testCopyCPUtoDefaultJit
|
|
- name: Run 10 MLPerf ResNet50 training steps (1 gpu)
|
|
if: ${{ matrix.dev == 'NV' }}
|
|
run: BENCHMARK_LOG=resnet_10steps MNISTMOCK=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py
|
|
- name: Run 10 MLPerf Bert training steps (1 gpu)
|
|
# TODO: remove BERT_LAYERS once scheduler is fast
|
|
run: BENCHMARK_LOG=bert_10steps CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=1 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
|
|
- name: Remote
|
|
run: |
|
|
pkill -f 'extra/remote/serve.py' || true
|
|
PYTHONPATH=. python3 extra/remote/serve.py 6482 &
|
|
sleep 1
|
|
DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 python3 test/test_tiny.py
|
|
if [[ "${{ matrix.dev }}" == "AMD" ]]; then
|
|
DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 AMD_AQL=1 python3 test/test_tiny.py
|
|
fi
|
|
pkill -f 'extra/remote/serve.py' || true
|
|
- name: Run process replay tests
|
|
uses: ./.github/actions/process-replay
|
|
|
|
llvmspeed:
|
|
name: LLVM Speed
|
|
runs-on: [self-hosted, Linux, tinyboxrandom]
|
|
timeout-minutes: 20
|
|
if: github.repository_owner == 'tinygrad'
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v6
|
|
- name: Speed Test
|
|
run: DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py
|
|
- name: Speed Test (BEAM=2)
|
|
run: BEAM=2 DEV=CPU:LLVM THREADS=0 python3 test/speed/external_test_speed_v_torch.py
|