diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5b69a36493..31169a23eb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -218,19 +218,19 @@ jobs: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids - name: UsbGPU boot time - run: sudo -E PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=AMD AMD_IFACE=USB time python3.11 test/test_tiny.py TestTiny.test_plus + run: sudo -E PYTHONPATH=. GMMU=0 DEBUG=2 AM_RESET=1 DEV=USB+AMD time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU tiny tests - run: sudo -E PYTHONPATH=. GMMU=0 DEV=AMD AMD_IFACE=USB python3.11 test/test_tiny.py + run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/test_tiny.py - name: UsbGPU copy speeds - run: sudo -E PYTHONPATH=. GMMU=0 DEV=AMD AMD_IFACE=USB python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds + run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD python3.11 test/external/external_test_usb_asm24.py TestDevCopySpeeds #- name: UsbGPU openpilot test - # run: sudo -E PYTHONPATH=. GMMU=0 DEV=AMD AMD_IFACE=USB GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx + # run: sudo -E PYTHONPATH=. GMMU=0 DEV=USB+AMD GRAPH_ONE_KERNEL=1 python3.11 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx - name: UsbGPU (USB4/TB) install script run: PYTHONPATH=. sh extra/setup_tinygpu_osx.sh - name: UsbGPU (USB4/TB) boot time - run: PYTHONPATH=. DEBUG=3 DEV=NV:NAK NV_IFACE=PCI time python3.11 test/test_tiny.py TestTiny.test_plus + run: PYTHONPATH=. DEBUG=3 DEV=PCI+NV:NAK time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU (USB4/TB) tiny tests - run: PYTHONPATH=. DEV=NV:NAK NV_IFACE=PCI python3.11 test/test_tiny.py + run: PYTHONPATH=. DEV=PCI+NV:NAK python3.11 test/test_tiny.py testnvidiabenchmark: name: tinybox green Benchmark @@ -667,9 +667,9 @@ jobs: echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal - name: openpilot compile3 0.10.1 driving_vision - run: BENCHMARK_LOG=usbgpu_openpilot_0_10_1_vision PYTHONPATH="." GMMU=0 DEV=AMD:LLVM AMD_IFACE=USB ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx + run: BENCHMARK_LOG=usbgpu_openpilot_0_10_1_vision PYTHONPATH="." GMMU=0 DEV=USB+AMD:LLVM ASSERT_MIN_STEP_TIME=50 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx - name: openpilot load_pickle 0.10.1 driving_vision - run: BENCHMARK_LOG=usbgpu_openpilot_0_10_1_vision_load_pickle PYTHONPATH="." GMMU=0 DEV=AMD AMD_IFACE=USB ASSERT_MIN_LOAD_TIME=15 python3 examples/openpilot/load_pickle.py + run: BENCHMARK_LOG=usbgpu_openpilot_0_10_1_vision_load_pickle PYTHONPATH="." GMMU=0 DEV=USB+AMD ASSERT_MIN_LOAD_TIME=15 python3 examples/openpilot/load_pickle.py testreddriverbenchmark: name: AM Benchmark @@ -737,8 +737,8 @@ jobs: pkill -f 'extra/remote/serve.py' || true PYTHONPATH=. python3 extra/remote/serve.py 6482 & sleep 1 - DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 DEV=AMD AMD_IFACE=PCI python3 test/test_tiny.py - DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 DEV=AMD AMD_AQL=1 AMD_IFACE=PCI python3 test/test_tiny.py + DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 DEV=PCI+AMD python3 test/test_tiny.py + DEBUG=2 PYTHONPATH=. REMOTE=127.0.0.1:6482 AM_RESET=1 DEV=PCI+AMD AMD_AQL=1 python3 test/test_tiny.py pkill -f 'extra/remote/serve.py' || true - name: Run process replay tests run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && PYTHONPATH=. python3 process_replay.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 671cfbcec0..90e135c2a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -690,9 +690,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 env: - DEV: AMD + DEV: PCI+AMD MOCKGPU: 1 - AMD_IFACE: PCI steps: - name: Checkout Code uses: actions/checkout@v6 @@ -705,7 +704,7 @@ jobs: - name: Run test_tiny on MOCKAM run: python test/test_tiny.py - name: Run test_tiny on MOCKAM USB - run: GMMU=0 AMD_IFACE=USB python test/test_tiny.py + run: GMMU=0 DEV=USB+AMD python test/test_tiny.py - name: Run test_hcq on MOCKAM run: python -m pytest test/device/test_hcq.py - name: Run disk copy tests on MOCKAM diff --git a/docs/env_vars.md b/docs/env_vars.md index d8dcf46cd1..20d6858c9b 100644 --- a/docs/env_vars.md +++ b/docs/env_vars.md @@ -28,17 +28,10 @@ The columns of this list are are: Variable, Possible Value(s) and Description. These control the behavior of core tinygrad even when used as a library. -### DEV variable - -The `DEV` variable deserves special note due to its more nuanced syntax. -`DEV` is used to specify the target device and target renderer for said device, separated by colons. -Specifying the renderer is optional, omitting a preference will cause tinygrad to automatically select a renderer from those -available on the system. Some example values for `DEV` are: `AMD`, `AMD:LLVM`, `NV:PTX`, etc. - Variable | Possible Value(s) | Description ---|---|--- DEBUG | [1-7] | enable debugging output (operations, timings, speed, generated code and more) -DEV | [AMD, NV, ...] | enable a specific backend +DEV | [AMD, NV, ...] | enable a specific backend, see [below](#dev-variable) BEAM | [#] | number of beams in kernel beam search DEFAULT_FLOAT | [HALF, ...]| specify the default float dtype (FLOAT32, HALF, BFLOAT16, FLOAT64, ...), default to FLOAT32 IMAGE | [1-2] | enable 2d specific optimizations @@ -50,7 +43,23 @@ ALLOW_TF32 | [1] | enable TensorFloat-32 tensor cores on Ampere WEBGPU_BACKEND | [WGPUBackendType_Metal, ...] | Force select a backend for WebGPU (Metal, DirectX, OpenGL, Vulkan...) CUDA_PATH | str | Use `CUDA_PATH/include` for CUDA headers for CUDA and NV backends. If not set, TinyGrad will use `/usr/local/cuda/include`, `/usr/include` and `/opt/cuda/include`. -## Debug breakdown +### DEV variable + +The `DEV` variable deserves special note due to its more nuanced syntax. +`DEV` is used to specify the target device, target renderer and target architecture for said device, separated by colons. +Specifying the renderer and architecture is optional, omitting a preference will cause tinygrad to automatically determine a suitable setting. +The `DEV` variable may also be used to specify the interface through which to access the device (eg. `PCI`, `USB`). Interfaces may be specified preceding the target triple, +separated by a plus (eg. `DEV=USB+AMD:LLVM`). Similarly as above, the interface may be omitted. Example usage follows: + +`DEV` contents | Interpretation +--- | --- +AMD | use the AMD device +AMD:LLVM | use the AMD device with the LLVM renderer +NV:CUDA:sm_70 | use the NV device with the CUDA renderer targetting sm_70 +AMD::gfx950 | use the AMD device targetting gfx950 +USB+AMD | use the AMD device over the USB interface + +### Debug breakdown Variable | Value | Description ---|---|--- diff --git a/docs/runtime.md b/docs/runtime.md index 81b4ce9cac..257ad0ff0c 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -4,8 +4,8 @@ tinygrad supports various runtimes, enabling your code to scale across a wide ra | Runtime | Description | Compiler Options | Requirements | |---------|-------------|------------------|--------------| -| [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | nvrtc (default)
PTX (`DEV=NV:PTX`) | Ampere/Ada/Blackwell series GPUs.
You can select an interface via `NV_IFACE=(NVK\|PCI)`. See [NV interfaces](#nv-interfaces) for details. | -| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`DEV=AMD:LLVM`)
HIP/COMGR (`DEV=AMD:HIP`) | RDNA2 or newer GPUs.
You can select an interface via `AMD_IFACE=(KFD\|PCI\|USB)`. See [AMD interfaces](#amd-interfaces) for details. | +| [NV](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_nv.py) | Provides acceleration for NVIDIA GPUs | nvrtc (default)
PTX (`DEV=NV:PTX`) | Ampere/Ada/Blackwell series GPUs.
You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [NV interfaces](#nv-interfaces) for details. | +| [AMD](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_amd.py) | Provides acceleration for AMD GPUs | LLVM (`DEV=AMD:LLVM`)
HIP/COMGR (`DEV=AMD:HIP`) | RDNA2 or newer GPUs.
You can select an interface via [the `DEV` variable](env_vars.md#dev-variable). See [AMD interfaces](#amd-interfaces) for details. | | [QCOM](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_qcom.py) | Provides acceleration for QCOM GPUs | - | 6xx series GPUs | | [METAL](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_metal.py) | Utilizes Metal for acceleration on Apple devices | - | M1+ Macs; Metal 3.0+ for `bfloat` support | | [CUDA](https://github.com/tinygrad/tinygrad/tree/master/tinygrad/runtime/ops_cuda.py) | Utilizes CUDA for acceleration on NVIDIA GPUs | nvrtc (default)
PTX (`DEV=CUDA:PTX`) | NVIDIA GPU with CUDA support | @@ -72,7 +72,7 @@ AMD backend supports several interfaces for communicating with devices: * `PCI`: uses the [AM driver](developer/am.md) * `USB`: USB3 interface for asm24xx chips. -You can force an interface by setting `AMD_IFACE` to one of these values. In the case of `AMD_IFACE=PCI`, this may unbind your GPU from the amdgpu driver. +You can force an interface by setting the interface component of [the `DEV` environment variable](env_vars.md#dev-variable) to one of these values. When set to `PCI`, this may unbind your GPU from the amdgpu driver. ## NV Interfaces NV backend supports several interfaces for communicating with devices: diff --git a/test/device/test_hcq.py b/test/device/test_hcq.py index 193073a99f..c14ca498a2 100644 --- a/test/device/test_hcq.py +++ b/test/device/test_hcq.py @@ -1,6 +1,6 @@ import unittest, ctypes, struct, os, random, numpy as np, time from tinygrad import Device, Tensor, dtypes -from tinygrad.helpers import getenv, mv_address, DEBUG +from tinygrad.helpers import getenv, mv_address, DEBUG, DEV from test.helpers import slow from tinygrad.device import Buffer, BufferSpec from tinygrad.runtime.support.hcq import HCQCompiled, HCQBuffer @@ -76,7 +76,7 @@ class TestHCQ(unittest.TestCase): TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value) TestHCQ.d0.timeline_value += 1 - @unittest.skipIf(Device.DEFAULT in {"CPU"} or getenv("AMD_IFACE", "") == "PCI", "Can't handle async update on CPU/MOCKAM device") + @unittest.skipIf(Device.DEFAULT in {"CPU"} or (DEV.interface == "PCI" and DEV.device == "AMD"), "Can't handle async update on CPU/MOCKAM device") def test_wait_late_set(self): for queue_type in [TestHCQ.d0.hw_compute_queue_t, TestHCQ.d0.hw_copy_queue_t]: if queue_type is None: continue @@ -575,7 +575,7 @@ class TestHCQ(unittest.TestCase): np.testing.assert_equal(cpu_buffer.numpy(), local_buf.numpy(), "failed") - @unittest.skipUnless(MOCKGPU and getenv("AMD_IFACE", "") != "PCI", "Emulate this on MOCKGPU to check the path in CI") + @unittest.skipUnless(MOCKGPU and not (DEV.device == "AMD" and DEV.interface == "PCI"), "Emulate this on MOCKGPU to check the path in CI") def test_on_device_hang(self): if not hasattr(self.d0, 'on_device_hang'): self.skipTest("device does not have on_device_hang") diff --git a/test/mockgpu/mockgpu.py b/test/mockgpu/mockgpu.py index 80c4d8d4c8..27debebeeb 100644 --- a/test/mockgpu/mockgpu.py +++ b/test/mockgpu/mockgpu.py @@ -1,5 +1,5 @@ import ctypes, ctypes.util, time, os, builtins, fcntl -from tinygrad.helpers import getenv +from tinygrad.helpers import DEV from tinygrad.runtime.support.hcq import FileIOInterface from test.mockgpu.nv.nvdriver import NVDriver from test.mockgpu.amd.amddriver import AMDDriver @@ -11,7 +11,7 @@ libc = ctypes.CDLL(ctypes.util.find_library("c")) libc.mmap.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_long] libc.mmap.restype = ctypes.c_void_p -_amd_iface = getenv("AMD_IFACE", "") +_amd_iface = DEV.target("AMD").interface drivers = [NVDriver(), AMDriver() if _amd_iface == "PCI" else (AMUSBDriver() if _amd_iface == "USB" else AMDDriver())] tracked_fds = {} diff --git a/test/null/test_device.py b/test/null/test_device.py index 94e9236541..e149ff273c 100644 --- a/test/null/test_device.py +++ b/test/null/test_device.py @@ -115,7 +115,8 @@ class TestDevVar(unittest.TestCase): def test_parse(self): for d, t in [("AMD", Target(device="AMD", renderer="")), ("AMD:LLVM", Target(device="AMD", renderer="LLVM")), (":LLVM", Target(device="", renderer="LLVM")), ("AMD::gfx1100", Target(device="AMD", arch="gfx1100")), - ("AMD:LLVM:gfx1100", Target(device="AMD", renderer="LLVM", arch="gfx1100")), ("::gfx1100", Target(arch="gfx1100"))]: + ("AMD:LLVM:gfx1100", Target(device="AMD", renderer="LLVM", arch="gfx1100")), ("::gfx1100", Target(arch="gfx1100")), + ("USB+", Target(interface="USB")), ("USB+AMD", Target(device="AMD", interface="USB"))]: with Context(DEV=d): self.assertEqual(DEV.value, t) self.assertEqual(str(DEV.value), d) diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index ac80d38bea..16947a686a 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -4,7 +4,7 @@ START_TIME = time.perf_counter() import os, functools, platform, re, contextlib, operator, hashlib, pickle, sqlite3, tempfile, pathlib, string, ctypes, sys, gzip, getpass, gc from collections import defaultdict import subprocess, shutil, math, types, copyreg, inspect, importlib, decimal, itertools -from dataclasses import dataclass, field, replace, asdict +from dataclasses import dataclass, field, replace from typing import ClassVar, Iterable, Any, TypeVar, Callable, Sequence, TypeGuard, Iterator, Generic, Generator, cast, overload T = TypeVar("T") @@ -179,13 +179,19 @@ class Target: device: str = "" renderer: str = "" arch: str = "" + interface: str = "" @staticmethod def parse(s:str) -> Target: - parts = [x.upper() if i < 2 else x for i,x in enumerate(s.split(':'))] - assert len(parts) <= 3, f"too many colons in target string: {s!r}" - return Target(*parts) - def __repr__(self) -> str: return re.sub(":*$", "", ":".join(asdict(self).values())) + if len(iface_split:=s.split('+')) == 2: iface, s = iface_split + elif len(iface_split) > 2: raise RuntimeError(f"too many '+' in target string: {s!r}") + else: iface = "" + match [x.upper() if i < 2 else x for i,x in enumerate(s.split(':'))]: + case [dev, ren, arch]: return Target(dev, ren, arch, iface) + case [dev, ren]: return Target(dev, ren, interface=iface) + case [dev]: return Target(dev, interface=iface) + case _: raise RuntimeError(f"too many ':' in target string: {s!r}") + def __repr__(self): return re.sub(":*$", "", (self.interface + "+" if self.interface else "") + ":".join([self.device, self.renderer, self.arch])) # replaces if not already set def replacedefault(self, **kwargs) -> Target: return replace(self, **{k:v for k,v in kwargs.items() if not getattr(self, k)}) diff --git a/tinygrad/runtime/support/hcq.py b/tinygrad/runtime/support/hcq.py index 79b8a48b72..e2876edd61 100644 --- a/tinygrad/runtime/support/hcq.py +++ b/tinygrad/runtime/support/hcq.py @@ -1,9 +1,10 @@ from __future__ import annotations from typing import cast, Callable, Type, TypeVar, Generic, Any import contextlib, decimal, statistics, time, ctypes, array, os, struct, collections, functools, itertools +from dataclasses import replace try: import fcntl # windows misses that except ImportError: fcntl = None #type:ignore[assignment] -from tinygrad.helpers import PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, select_first_inited, unwrap, suppress_finalizing +from tinygrad.helpers import DEV, PROFILE, getenv, to_mv, from_mv, cpu_profile, ProfileRangeEvent, select_first_inited, unwrap, suppress_finalizing from tinygrad.helpers import TracingKey from tinygrad.device import Device, BufferSpec, Compiled, LRUAllocator, ProfileDeviceEvent, ProfileProgramEvent from tinygrad.uop.ops import sym_infer, sint, UOp @@ -485,9 +486,12 @@ class HCQCompiled(Compiled, Generic[SignalType]): return buf, realloced def _select_iface(self, *ifaces:Type): - if val:=getenv(f'{type(self).__name__[:-6].upper()}_IFACE', ""): ifaces = tuple(x for x in ifaces if x.__name__.startswith(val.upper())) + assert (v:=getenv(k:=f'{type(self).__name__[:-6].upper()}_IFACE', "")) == "", \ + f"{k}={v} is deprecated, use DEV={replace(DEV.target(type(self).__name__[:-6]), interface=v)} instead" + if (iface:=DEV.target(dev:=type(self).__name__[:-6]).interface): ifaces = tuple(x for x in ifaces if x.__name__.startswith(iface.upper())) + assert len(ifaces), f"No interface for {dev} " + (f"matches request {iface!r}" if iface else "is available") return select_first_inited([functools.partial(cast(Callable, iface), self, self.device_id) for iface in ifaces], - f"No interface for {type(self).__name__[:-6]}:{self.device_id} is available") + f"No interface for {dev}:{self.device_id} is available") def _is_cpu(self) -> bool: return hasattr(self, 'device') and self.device.split(":")[0] == "CPU"