From dabb02767f37f2e33af30a102afa6e60da9736b6 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Thu, 20 Nov 2025 23:19:11 -0800 Subject: [PATCH] set AMD profile mode with sudo on SQTT or PMC (#13403) * require profile mode * add mode setter * cleanup * not needed * SQTT_LIMIT_SE --- extra/sqtt/active_sqtt_parse.py | 7 ------- tinygrad/runtime/ops_amd.py | 34 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/extra/sqtt/active_sqtt_parse.py b/extra/sqtt/active_sqtt_parse.py index 6cbe5fc1eb..a803b807d0 100644 --- a/extra/sqtt/active_sqtt_parse.py +++ b/extra/sqtt/active_sqtt_parse.py @@ -15,13 +15,6 @@ from tinygrad.device import Device, ProfileDeviceEvent from extra.sqtt.attempt_sqtt_parse import parse_sqtt_print_packets -# TODO: should really check for AM driver / USB -if not OSX: - def set_power(x): system(f"sudo /opt/rocm/bin/amd-smi set -l {x}") - @atexit.register - def reset_power(): set_power("auto") - set_power("stable_std") - dev = Device["AMD"] @contextlib.contextmanager diff --git a/tinygrad/runtime/ops_amd.py b/tinygrad/runtime/ops_amd.py index df58228794..341727ee6a 100644 --- a/tinygrad/runtime/ops_amd.py +++ b/tinygrad/runtime/ops_amd.py @@ -1,6 +1,6 @@ from __future__ import annotations from typing import cast, ClassVar -import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections +import os, ctypes, struct, hashlib, functools, importlib, mmap, errno, array, contextlib, sys, weakref, itertools, collections, atexit assert sys.platform != 'win32' from dataclasses import dataclass from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQBuffer, HWQueue, CLikeArgsState, HCQSignal, HCQProgram, FileIOInterface @@ -20,7 +20,8 @@ from tinygrad.runtime.support.amd import AMDReg, AMDIP, import_module, import_so from tinygrad.runtime.support.system import System, PCIIfaceBase, PCIAllocationMeta, PCIDevice, USBPCIDevice, MAP_FIXED, MAP_NORESERVE if getenv("IOCTL"): import extra.hip_gpu_driver.hip_ioctl # noqa: F401 # pylint: disable=unused-import -SQTT, SQTT_ITRACE_SE_MASK, PMC = ContextVar("SQTT", VIZ.value>=2), ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("PMC", 0) +SQTT, SQTT_ITRACE_SE_MASK, SQTT_LIMIT_SE = ContextVar("SQTT", VIZ.value>=2), ContextVar("SQTT_ITRACE_SE_MASK", 0b11), ContextVar("SQTT_LIMIT_SE", 1) +PMC = ContextVar("PMC", 0) EVENT_INDEX_PARTIAL_FLUSH = 4 # based on a comment in nvd.h WAIT_REG_MEM_FUNCTION_EQ = 3 # == WAIT_REG_MEM_FUNCTION_NEQ = 4 # != @@ -193,11 +194,12 @@ class AMDComputeQueue(HWQueue): bind_point=(__BIND_POINT_COMPUTE:=1), api_pso_hash=data64_le(prg.libhash[0]))) self.sqtt_userdata(sqtt.struct_rgp_sqtt_marker_event(has_thread_dims=1, cmd_id=next(prg.dev.sqtt_next_cmd_id)), *global_size) - se_cap = max(prod([x if isinstance(x, int) else 1 for x in global_size]) // 4, 1) // 32 - for xcc in range(self.dev.xccs): - with self.pred_exec(xcc_mask=1 << xcc): - for i in range(8 if prg.dev.target >= (11,0,0) else 4): - self.wreg(getattr(self.gc, f'regCOMPUTE_STATIC_THREAD_MGMT_SE{i}'), min(0xffffffff, (1 << (se_cap + (1 if i == 0 else 0))) - 1)) + if SQTT_LIMIT_SE: + se_cap = max(prod([x if isinstance(x, int) else 1 for x in global_size]) // 4, 1) // 32 + for xcc in range(self.dev.xccs): + with self.pred_exec(xcc_mask=1 << xcc): + for i in range(8 if prg.dev.target >= (11,0,0) else 4): + self.wreg(getattr(self.gc, f'regCOMPUTE_STATIC_THREAD_MGMT_SE{i}'), min(0xffffffff, (1 << (se_cap + (1 if i == 0 else 0))) - 1)) def sqtt_userdata(self, data, *extra_dwords): data_ints = [x[0] for x in struct.iter_unpack(' /dev/null")) + os.system(f"echo 'profile_standard' | sudo tee {fn} > /dev/null") + self.require_profile_mode(can_set_mode=False) + else: + raise RuntimeError("PMC/SQTT requires stable power state: run `amd-smi set -l stable_std` for KFD iface") class PCIIface(PCIIfaceBase): gpus:ClassVar[list[str]] = [] @@ -788,7 +798,7 @@ class PCIIface(PCIIfaceBase): self._setup_adev(self.pci_dev) self.pci_dev.write_config(pci.PCI_COMMAND, self.pci_dev.read_config(pci.PCI_COMMAND, 2) | pci.PCI_COMMAND_MASTER, 2) - def is_in_profile_mode(self): return True + def require_profile_mode(self): return True def _setup_adev(self, pci_dev:PCIDevice, dma_regions:list[tuple[int, MMIOInterface]]|None=None): self.dev_impl:AMDev = AMDev(pci_dev, dma_regions) @@ -925,7 +935,7 @@ class AMDDevice(HCQCompiled): self.pmc_enabled = PROFILE and PMC > 0 if self.pmc_enabled: if self.target[0] not in {9, 11, 12}: raise RuntimeError(f'PMC are not supported on gc:{self.target}') - if not self.iface.is_in_profile_mode(): raise RuntimeError("PMC requires stable power state: run `amd-smi set -l stable_std` for KFD iface") + self.iface.require_profile_mode() self.pmc_sched:list[PMCSample] = [] self.pmc_counters = import_pmc(self.target) @@ -943,7 +953,7 @@ class AMDDevice(HCQCompiled): self.sqtt_enabled = PROFILE and SQTT > 0 if self.sqtt_enabled: if self.target[0] not in {9, 11, 12}: raise RuntimeError(f'SQ Thread Tracing is not supported on gc:{self.target}') - if not self.iface.is_in_profile_mode(): raise RuntimeError("SQTT requires stable power state: run `amd-smi set -l stable_std` for KFD iface") + self.iface.require_profile_mode() SQTT_BUFFER_SIZE = getenv("SQTT_BUFFER_SIZE", 256) # in mb, per shader engine self.sqtt_buffers = [self.allocator.alloc(SQTT_BUFFER_SIZE << 20, BufferSpec(nolru=True, uncached=True)) for _ in range(self.se_cnt)]