From 544928766d551df01da11dcada23fe9bf7dc8201 Mon Sep 17 00:00:00 2001 From: nimlgen <138685161+nimlgen@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:00:28 +0300 Subject: [PATCH] hcq_smi: kill mac pids (#14398) --- .github/workflows/benchmark.yml | 24 ++++++++++++++---------- extra/hcq/hcq_smi.py | 20 ++++++-------------- tinygrad/runtime/support/system.py | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d2329a822b..d0edf2d167 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -145,6 +145,10 @@ jobs: run: | echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal + - name: Kill stale pids + run: | + PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids + PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids - name: UsbGPU boot time run: sudo -E PYTHONPATH=. DEBUG=2 AM_RESET=1 AMD=1 AMD_IFACE=USB time python3.11 test/test_tiny.py TestTiny.test_plus - name: UsbGPU tiny tests @@ -332,9 +336,9 @@ jobs: - name: Setcap to python run: ./extra/amdpci/setup_python_cap.sh - name: Remove amd modules - run: ./extra/hcq/hcq_smi.py amd rmmod + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod - name: Kill stale pids - run: ./extra/hcq/hcq_smi.py amd kill_pids + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids #- name: Insert amdgpu # run: sudo modprobe amdgpu - name: Symlink models and datasets @@ -444,9 +448,9 @@ jobs: - name: Setcap to python run: ./extra/amdpci/setup_python_cap.sh - name: Remove amd modules - run: ./extra/hcq/hcq_smi.py amd rmmod + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod - name: Kill stale pids - run: ./extra/hcq/hcq_smi.py amd kill_pids + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids - name: Symlink models and datasets run: | mkdir -p weights @@ -496,9 +500,9 @@ jobs: - name: Setcap to python run: ./extra/amdpci/setup_python_cap.sh - name: Remove amd modules - run: ./extra/hcq/hcq_smi.py amd rmmod + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod - name: Kill stale pids - run: ./extra/hcq/hcq_smi.py amd kill_pids + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids - name: Symlink models and datasets run: | mkdir -p weights @@ -587,9 +591,9 @@ jobs: - name: Setcap to python run: ./extra/amdpci/setup_python_cap.sh - name: Remove amd modules - run: ./extra/hcq/hcq_smi.py amd rmmod + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod - name: Kill stale pids - run: ./extra/hcq/hcq_smi.py amd kill_pids + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids - name: Symlink models and datasets run: | mkdir -p weights @@ -651,9 +655,9 @@ jobs: - name: Setcap to python run: ./extra/amdpci/setup_python_cap.sh - name: Remove nv modules - run: ./extra/hcq/hcq_smi.py nv rmmod + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py nv rmmod - name: Kill stale pids - run: ./extra/hcq/hcq_smi.py nv kill_pids + run: PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids - name: Symlink models and datasets run: | mkdir -p weights diff --git a/extra/hcq/hcq_smi.py b/extra/hcq/hcq_smi.py index dc883e3f3f..edb465235e 100755 --- a/extra/hcq/hcq_smi.py +++ b/extra/hcq/hcq_smi.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 import argparse, glob, os, time, subprocess, sys +from tinygrad.helpers import temp def scan_devs_based_on_lock(prefix:str, args) -> list[str]: target_dev = args.pci_bus if 'pci_bus' in args.__dir__() else "" devs = [] - for dev in glob.glob(f'/tmp/{prefix}_*.lock'): - dev_id = dev[8:-5] - if os.path.exists(f"/sys/bus/pci/devices/{dev_id}") and dev_id.startswith(target_dev): devs.append(dev_id) + for dev in glob.glob(temp(f'{prefix}_*.lock')): + dev_id = dev.split('/')[-1][len(prefix)+1:-5] + if dev_id.startswith(target_dev): devs.append(dev_id) return devs def _do_reset_device(pci_bus): os.system(f"sudo sh -c 'echo 1 > /sys/bus/pci/devices/{pci_bus}/reset'") @@ -53,16 +54,7 @@ def cmd_show_pids(args): for dev in devs: try: - pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1] - print(f"{dev}: {pid}") - except subprocess.CalledProcessError: print(f"{dev}: No processes found using this device") - -def cmd_kill_pids(args): - devs = scan_devs_based_on_lock(prefix:={"amd":"am", "nv":"nv"}[args.backend], args) - - for dev in devs: - try: - pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1] + pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] print(f"{dev}: {pid}") except subprocess.CalledProcessError: print(f"{dev}: No processes found using this device") @@ -74,7 +66,7 @@ def cmd_kill_pids(args): if i > 0: time.sleep(0.2) try: - try: pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1] + try: pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1] except subprocess.CalledProcessError: break print(f"Killing process {pid} (which uses {dev})") diff --git a/tinygrad/runtime/support/system.py b/tinygrad/runtime/support/system.py index 2388c7cef4..04ad686788 100644 --- a/tinygrad/runtime/support/system.py +++ b/tinygrad/runtime/support/system.py @@ -131,7 +131,7 @@ class _System: else: self.lock_fd = os.open(lock_name, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o666) try: fcntl.flock(self.lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: raise RuntimeError(f"Failed to take lock file {name}. It's already in use.") + except OSError: raise RuntimeError(f"Failed to acquire lock file {name}. `sudo lsof {lock_name}` may help identify the process holding the lock.") return self.lock_fd