tests pass

.github/workflows/test.yml

@@ -5,7 +5,6 @@ env:
   CAPTURE_PROCESS_REPLAY: 1
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PYTHONPATH: ${{ github.workspace }}
-  IGNORE_OOB: 0
 
 on:
   push:
@@ -37,8 +36,6 @@ jobs:
     name: Docs
     runs-on: ubuntu-22.04
     timeout-minutes: 10
-    env:
-      IGNORE_OOB: 1
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
@@ -105,11 +102,15 @@ jobs:
       run: |
         sudo apt update || true
         sudo apt install -y --no-install-recommends ninja-build
+    - name: Lint with ruff
+      run: |
+        pip3 install --upgrade --force-reinstall ruff==0.11.0
+        python3 -m ruff check extra/torch_backend/backend.py
     - name: Test one op
       run: FORWARD_ONLY=1 TINY_BACKEND=1 python3 test/test_ops.py TestOps.test_add
     - name: Test ResNet-18
       run: DEBUG=2 python3 extra/torch_backend/example.py
-    - name: custom tests
+    - name: My (custom) tests
       run: python3 extra/torch_backend/test.py
     - name: Test one op in torch tests
       run: DEBUG=2 python3 extra/torch_backend/torch_tests.py TestTinyBackendPRIVATEUSE1.test_unary_log_tiny_float32
@@ -232,11 +233,10 @@ jobs:
       run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string='  ' --recursive=y .
     - name: Lint with ruff
       run: |
-        pip3 install --upgrade --force-reinstall ruff==0.14.10
+        pip3 install --upgrade --force-reinstall ruff==0.11.0
         python3 -m ruff check .
         python3 -m ruff check examples/mlperf/ --ignore E501
         python3 -m ruff check extra/thunder/tiny/ --ignore E501 --ignore F841 --ignore E722
-        python3 -m ruff check extra/torch_backend/backend.py
     - name: Run mypy
       run: |
         python -m mypy --strict-equality --lineprecision-report .
@@ -310,7 +310,7 @@ jobs:
         deps: testing_unit
         python-version: '3.14'
     - name: Test SPEC=2
-      run: SPEC=2 pytest --maxfail=10 -n auto --durations=30 --ignore=test/models --ignore test/test_custom_kernel.py --ignore test/unit/test_hashing.py --timeout 60 -k "not test_setitem_big" --splits 2 --group ${{ matrix.group }}
+      run: IGNORE_OOB=0 SPEC=2 PYTHONPATH="." pytest --maxfail=10 -n auto --durations=30 --ignore=test/models --ignore test/test_custom_kernel.py --ignore test/unit/test_hashing.py --timeout 60 -k "not test_setitem_big" --splits 2 --group ${{ matrix.group }}
 
   fuzzing:
     name: Fuzzing
@@ -473,8 +473,6 @@ jobs:
     name: Test LLM
     runs-on: ubuntu-24.04
     timeout-minutes: 15
-    env:
-      IGNORE_OOB: 1
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -656,7 +654,7 @@ jobs:
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
 
-  testamdasm:
+  testrdna3:
     name: AMD ASM IDE
     runs-on: ubuntu-24.04
     timeout-minutes: 10
@@ -679,28 +677,12 @@ jobs:
         run: cloc --by-file extra/assembly/amd/*.py
       - name: Run RDNA3 emulator tests
         run: python -m pytest -n=auto extra/assembly/amd/ --durations 20
-      - name: Run RDNA3 emulator tests (AMD_LLVM=1)
-        run: AMD_LLVM=1 python -m pytest -n=auto extra/assembly/amd/ --durations 20
-      - name: Run RDNA3 dtype tests
-        run: AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=0 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py
-      - name: Run RDNA3 dtype tests (AMD_LLVM=1)
-        run: AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=1 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py
-
-  testamdautogen:
-    name: AMD autogen
-    runs-on: ubuntu-24.04
-    timeout-minutes: 10
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-      - name: Setup Environment
-        uses: ./.github/actions/setup-tinygrad
-        with:
-          key: rdna3-autogen
-          pydeps: "pdfplumber"
+      - name: Install pdfplumber
+        run: pip install pdfplumber
       - name: Verify AMD autogen is up to date
         run: |
-          python -m extra.assembly.amd.pdf --arch all
+          python -m extra.assembly.amd.dsl --arch all
+          python -m extra.assembly.amd.pcode --arch all
           git diff --exit-code extra/assembly/amd/autogen/
 
   testnvidia:

CLAUDE.md

@@ -208,9 +208,3 @@ Key patterns to watch (from ResNet50 benchmark):
 - `vmin==vmax folding`: ~55ms, 0.33% match rate - checks 52K ops but rarely matches
 
 Patterns with 0% match rate are workload-specific overhead. They may be useful in other workloads, so don't remove them without understanding their purpose.
-
-## AMD Performance Counter Profiling
-
-Set VIZ to `-2` to save performance counters traces for the AMD backend.
-
-Use the CLI in `./extra/sqtt/roc.py` to explore the trace.

extra/assembly/amd/README

@@ -1,31 +0,0 @@
-An integrated environment for AMD GPU assembly and emulation
-
-Test with `PYTHONPATH="." pytest -n12 extra/assembly/amd/`
-`AMD_LLVM=1 PYTHONPATH="." pytest -n12 extra/assembly/amd/`
-
-* pdf.py -- extract assembly format + instruction psuedocode from AMD PDF
-* dsl.py -- helpers for the autogen instruction classes in `__init__.py`. should be standalone with init
-* pcode.py -- psuedocode execution environment. psuedocode should be transformed as little as possible.
-* asm.py -- an asm/disasm function to transform to and from AMD assembly syntax
-* emu.py -- an emulator for RDNA that runs in tinygrad with `AMD=1 MOCKGPU=1 PYTHON_REMU=1`
-
-The code should be as readable and deduplicated as possible. asm and emu shouldn't be required for dsl.
-
-test_emu.py has a good set of instruction tests for the emulation, with USE_HW=1 it will compare to real hardware.
-Whenever an instruction is fixed, regression tests should be added here and confirmed with real hardware.
-
-test_llvm.py tests asm/disasm on the LLVM tests, confirming it behaves the same as LLVM.
-
-tinygrad's dtype tests should pass with and without LLVM. they run in about 12 seconds.
-
-`PYTHONPATH="." AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=0 pytest -n=12 test/test_dtype_alu.py test/test_dtype.py`
-`PYTHONPATH="." AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=1 pytest -n=12 test/test_dtype_alu.py test/test_dtype.py`
-
-The ops tests also pass, but they are very slow, so you should run them one at a time.
-
-`SKIP_SLOW_TEST=1 PYTHONPATH="." AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=0 pytest -n=12 test/test_ops.py`
-`SKIP_SLOW_TEST=1 PYTHONPATH="." AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=1 pytest -n=12 test/test_ops.py`
-
-When something is caught by main tinygrad tests, a local regression test should be added to `extra/assembly/amd/test`. While working with tinygrad, you can dump the assembly with `DEBUG=7`. These tests all pass on real hardware, so if a test is failing with `AMD=1 PYTHON_REMU=1 MOCKGPU=1` it's likely because an instruction is emulated incorrectly. You can test without `MOCKGPU=1` to test on real hardware, if it works on real hardware there's a bug in the emulator.
-
-Currently, only RDNA3 is well supported, but when finished, this will support RDNA3+RDNA4+CDNA in ~2000 lines. Count lines with `cloc --by-file extra/assembly/amd/*.py`

extra/assembly/amd/dsl.py

@@ -1,139 +1,27 @@
 # library for RDNA3 assembly DSL
 # mypy: ignore-errors
 from __future__ import annotations
-import struct, math, re
 from enum import IntEnum
-from functools import cache, cached_property
 from typing import overload, Annotated, TypeVar, Generic
-from extra.assembly.amd.autogen.rdna3.enum import (VOP1Op, VOP2Op, VOP3Op, VOP3SDOp, VOP3POp, VOPCOp, VOPDOp, SOP1Op, SOP2Op,
-  SOPCOp, SOPKOp, SOPPOp, SMEMOp, DSOp, FLATOp, MUBUFOp, MTBUFOp, MIMGOp, VINTERPOp)
-
-# Common masks and bit conversion functions
-MASK32, MASK64 = 0xffffffff, 0xffffffffffffffff
-_struct_f, _struct_I = struct.Struct("<f"), struct.Struct("<I")
-_struct_e, _struct_H = struct.Struct("<e"), struct.Struct("<H")
-_struct_d, _struct_Q = struct.Struct("<d"), struct.Struct("<Q")
-def _f32(i): return _struct_f.unpack(_struct_I.pack(i & MASK32))[0]
-def _i32(f):
-  if isinstance(f, int): f = float(f)
-  if math.isnan(f): return 0xffc00000 if math.copysign(1.0, f) < 0 else 0x7fc00000
-  if math.isinf(f): return 0x7f800000 if f > 0 else 0xff800000
-  try: return _struct_I.unpack(_struct_f.pack(f))[0]
-  except (OverflowError, struct.error): return 0x7f800000 if f > 0 else 0xff800000
-def _sext(v, b): return v - (1 << b) if v & (1 << (b - 1)) else v
-def _f16(i): return _struct_e.unpack(_struct_H.pack(i & 0xffff))[0]
-def _i16(f):
-  if math.isnan(f): return 0x7e00
-  if math.isinf(f): return 0x7c00 if f > 0 else 0xfc00
-  try: return _struct_H.unpack(_struct_e.pack(f))[0]
-  except (OverflowError, struct.error): return 0x7c00 if f > 0 else 0xfc00
-def _f64(i): return _struct_d.unpack(_struct_Q.pack(i & MASK64))[0]
-def _i64(f):
-  if math.isnan(f): return 0x7ff8000000000000
-  if math.isinf(f): return 0x7ff0000000000000 if f > 0 else 0xfff0000000000000
-  try: return _struct_Q.unpack(_struct_d.pack(f))[0]
-  except (OverflowError, struct.error): return 0x7ff0000000000000 if f > 0 else 0xfff0000000000000
-
-# Instruction spec - register counts and dtypes derived from instruction names
-_REGS = {'B32': 1, 'B64': 2, 'B96': 3, 'B128': 4, 'B256': 8, 'B512': 16,
-         'F32': 1, 'I32': 1, 'U32': 1, 'F64': 2, 'I64': 2, 'U64': 2,
-         'F16': 1, 'I16': 1, 'U16': 1, 'B16': 1, 'I8': 1, 'U8': 1, 'B8': 1}
-_CVT_RE = re.compile(r'CVT_([FIUB]\d+)_([FIUB]\d+)$')
-_MAD_MUL_RE = re.compile(r'(?:MAD|MUL)_([IU]\d+)_([IU]\d+)$')
-_PACK_RE = re.compile(r'PACK_([FIUB]\d+)_([FIUB]\d+)$')
-_DST_SRC_RE = re.compile(r'_([FIUB]\d+)_([FIUB]\d+)$')
-_SINGLE_RE = re.compile(r'_([FIUB](?:32|64|16|8|96|128|256|512))$')
-@cache
-def _suffix(name: str) -> tuple[str | None, str | None]:
-  name = name.upper()
-  if m := _CVT_RE.search(name): return m.group(1), m.group(2)
-  if m := _MAD_MUL_RE.search(name): return m.group(1), m.group(2)
-  if m := _PACK_RE.search(name): return m.group(1), m.group(2)
-  if m := _DST_SRC_RE.search(name): return m.group(1), m.group(2)
-  if m := _SINGLE_RE.search(name): return m.group(1), m.group(1)
-  return None, None
-_SPECIAL_REGS = {
-  'V_LSHLREV_B64': (2, 1, 2, 1), 'V_LSHRREV_B64': (2, 1, 2, 1), 'V_ASHRREV_I64': (2, 1, 2, 1),
-  'S_LSHL_B64': (2, 2, 1, 1), 'S_LSHR_B64': (2, 2, 1, 1), 'S_ASHR_I64': (2, 2, 1, 1),
-  'S_BFE_U64': (2, 2, 1, 1), 'S_BFE_I64': (2, 2, 1, 1), 'S_BFM_B64': (2, 1, 1, 1),
-  'S_BITSET0_B64': (2, 1, 1, 1), 'S_BITSET1_B64': (2, 1, 1, 1),
-  'S_BITCMP0_B64': (1, 2, 1, 1), 'S_BITCMP1_B64': (1, 2, 1, 1),
-  'V_LDEXP_F64': (2, 2, 1, 1), 'V_TRIG_PREOP_F64': (2, 2, 1, 1),
-  'V_CMP_CLASS_F64': (1, 2, 1, 1), 'V_CMPX_CLASS_F64': (1, 2, 1, 1),
-  'V_CMP_CLASS_F32': (1, 1, 1, 1), 'V_CMPX_CLASS_F32': (1, 1, 1, 1),
-  'V_CMP_CLASS_F16': (1, 1, 1, 1), 'V_CMPX_CLASS_F16': (1, 1, 1, 1),
-  'V_MAD_U64_U32': (2, 1, 1, 2), 'V_MAD_I64_I32': (2, 1, 1, 2),
-  'V_QSAD_PK_U16_U8': (2, 2, 1, 2), 'V_MQSAD_PK_U16_U8': (2, 2, 1, 2), 'V_MQSAD_U32_U8': (4, 2, 1, 4),
-}
-_SPECIAL_DTYPE = {
-  'V_LSHLREV_B64': ('B64', 'U32', 'B64', None), 'V_LSHRREV_B64': ('B64', 'U32', 'B64', None), 'V_ASHRREV_I64': ('I64', 'U32', 'I64', None),
-  'S_LSHL_B64': ('B64', 'B64', 'U32', None), 'S_LSHR_B64': ('B64', 'B64', 'U32', None), 'S_ASHR_I64': ('I64', 'I64', 'U32', None),
-  'S_BFE_U64': ('U64', 'U64', 'U32', None), 'S_BFE_I64': ('I64', 'I64', 'U32', None),
-  'S_BFM_B64': ('B64', 'U32', 'U32', None), 'S_BITSET0_B64': ('B64', 'U32', None, None), 'S_BITSET1_B64': ('B64', 'U32', None, None),
-  'S_BITCMP0_B64': ('SCC', 'B64', 'U32', None), 'S_BITCMP1_B64': ('SCC', 'B64', 'U32', None),
-  'V_LDEXP_F64': ('F64', 'F64', 'I32', None), 'V_TRIG_PREOP_F64': ('F64', 'F64', 'U32', None),
-  'V_CMP_CLASS_F64': ('VCC', 'F64', 'U32', None), 'V_CMPX_CLASS_F64': ('EXEC', 'F64', 'U32', None),
-  'V_CMP_CLASS_F32': ('VCC', 'F32', 'U32', None), 'V_CMPX_CLASS_F32': ('EXEC', 'F32', 'U32', None),
-  'V_CMP_CLASS_F16': ('VCC', 'F16', 'U32', None), 'V_CMPX_CLASS_F16': ('EXEC', 'F16', 'U32', None),
-  'V_MAD_U64_U32': ('U64', 'U32', 'U32', 'U64'), 'V_MAD_I64_I32': ('I64', 'I32', 'I32', 'I64'),
-  'V_QSAD_PK_U16_U8': ('B64', 'B64', 'B64', 'B64'), 'V_MQSAD_PK_U16_U8': ('B64', 'B64', 'B64', 'B64'),
-  'V_MQSAD_U32_U8': ('B128', 'B64', 'B64', 'B128'),
-}
-@cache
-def spec_regs(name: str) -> tuple[int, int, int, int]:
-  uname = name.upper()
-  if uname in _SPECIAL_REGS: return _SPECIAL_REGS[uname]
-  if 'SAD' in uname and 'U8' in uname and 'QSAD' not in uname and 'MQSAD' not in uname: return 1, 1, 1, 1
-  dst_suf, src_suf = _suffix(name)
-  return _REGS.get(dst_suf, 1), _REGS.get(src_suf, 1), _REGS.get(src_suf, 1), _REGS.get(src_suf, 1)
-@cache
-def spec_dtype(name: str) -> tuple[str | None, str | None, str | None, str | None]:
-  uname = name.upper()
-  if uname in _SPECIAL_DTYPE: return _SPECIAL_DTYPE[uname]
-  if 'SAD' in uname and ('U8' in uname or 'U16' in uname) and 'QSAD' not in uname and 'MQSAD' not in uname: return 'U32', 'U32', 'U32', 'U32'
-  if '_CMP_' in uname or '_CMPX_' in uname:
-    dst_suf, src_suf = _suffix(name)
-    return 'EXEC' if '_CMPX_' in uname else 'VCC', src_suf, src_suf, None
-  dst_suf, src_suf = _suffix(name)
-  return dst_suf, src_suf, src_suf, src_suf
-_F16_RE = re.compile(r'_[FIUB]16(?:_|$)')
-_F64_RE = re.compile(r'_[FIUB]64(?:_|$)')
-@cache
-def spec_is_16bit(name: str) -> bool:
-  uname = name.upper()
-  if 'SAD' in uname or 'PACK' in uname or '_PK_' in uname or 'SAT_PK' in uname or 'DOT2' in uname: return False
-  if '_F32' in uname or '_I32' in uname or '_U32' in uname or '_B32' in uname: return False
-  return bool(_F16_RE.search(uname))
-@cache
-def spec_is_64bit(name: str) -> bool: return bool(_F64_RE.search(name.upper()))
-_3SRC = {'FMA', 'MAD', 'MIN3', 'MAX3', 'MED3', 'DIV_FIX', 'DIV_FMAS', 'DIV_SCALE', 'SAD', 'LERP', 'ALIGN', 'CUBE', 'BFE', 'BFI',
-         'PERM_B32', 'PERMLANE', 'CNDMASK', 'XOR3', 'OR3', 'ADD3', 'LSHL_OR', 'AND_OR', 'LSHL_ADD', 'ADD_LSHL', 'XAD', 'MAXMIN',
-         'MINMAX', 'DOT2', 'DOT4', 'DOT8', 'WMMA', 'CVT_PK_U8', 'MULLIT', 'CO_CI'}
-_2SRC = {'FMAC'}  # FMAC uses dst as implicit accumulator, so only 2 explicit sources
-def spec_num_srcs(name: str) -> int:
-  name = name.upper()
-  if any(k in name for k in _2SRC): return 2
-  return 3 if any(k in name for k in _3SRC) else 2
-def is_dtype_16(dt: str | None) -> bool: return dt is not None and '16' in dt
-def is_dtype_64(dt: str | None) -> bool: return dt is not None and '64' in dt
 
 # Bit field DSL
 class BitField:
-  def __init__(self, hi: int, lo: int, name: str | None = None): self.hi, self.lo, self.name, self._marker = hi, lo, name, None
-  def __set_name__(self, owner, name):
-    import typing
-    self.name, self._owner = name, owner
-    # Cache marker at class definition time
-    hints = typing.get_type_hints(owner, include_extras=True)
-    if name in hints:
-      hint = hints[name]
-      if typing.get_origin(hint) is Annotated:
-        args = typing.get_args(hint)
-        self._marker = args[1] if len(args) > 1 else None
+  def __init__(self, hi: int, lo: int, name: str | None = None): self.hi, self.lo, self.name = hi, lo, name
+  def __set_name__(self, owner, name): self.name, self._owner = name, owner
   def __eq__(self, val: int) -> tuple[BitField, int]: return (self, val)  # type: ignore
   def mask(self) -> int: return (1 << (self.hi - self.lo + 1)) - 1
   @property
-  def marker(self) -> type | None: return self._marker
+  def marker(self) -> type | None:
+    # Get marker from Annotated type hint if present
+    import typing
+    if hasattr(self, '_owner') and self.name:
+      hints = typing.get_type_hints(self._owner, include_extras=True)
+      if self.name in hints:
+        hint = hints[self.name]
+        if typing.get_origin(hint) is Annotated:
+          args = typing.get_args(hint)
+          return args[1] if len(args) > 1 else None
+    return None
   @overload
   def __get__(self, obj: None, objtype: type) -> BitField: ...
   @overload
@@ -143,10 +31,6 @@ class BitField:
     val = unwrap(obj._values.get(self.name, 0))
     # Convert to IntEnum if marker is an IntEnum subclass
     if self.marker and isinstance(self.marker, type) and issubclass(self.marker, IntEnum):
-      # VOP3 with VOPC opcodes (0-255) -> VOPCOp, VOP3SD opcodes -> VOP3SDOp
-      if self.marker is VOP3Op:
-        if val < 256: return VOPCOp(val)
-        if val in Inst._VOP3SD_OPS: return VOP3SDOp(val)
       try: return self.marker(val)
       except ValueError: pass
     return val
@@ -229,36 +113,31 @@ def unwrap(val) -> int:
   if hasattr(val, 'idx'): return val.idx  # Reg
   return val
 
-# Encoding/decoding constants
+# Encoding helpers
 FLOAT_ENC = {0.5: 240, -0.5: 241, 1.0: 242, -1.0: 243, 2.0: 244, -2.0: 245, 4.0: 246, -4.0: 247}
-FLOAT_DEC = {v: str(k) for k, v in FLOAT_ENC.items()}
-SPECIAL_GPRS = {106: "vcc_lo", 107: "vcc_hi", 124: "null", 125: "m0", 126: "exec_lo", 127: "exec_hi", 253: "scc"}
-SPECIAL_PAIRS = {106: "vcc", 126: "exec"}
 SRC_FIELDS = {'src0', 'src1', 'src2', 'ssrc0', 'ssrc1', 'soffset', 'srcx0', 'srcy0'}
 RAW_FIELDS = {'vdata', 'vdst', 'vaddr', 'addr', 'data', 'data0', 'data1', 'sdst', 'sdata', 'vsrc1'}
 
-def _encode_reg(val: Reg) -> int: return (108 if isinstance(val, TTMP) else 0) + val.idx
-
-def _is_inline_const(v: int) -> bool: return 0 <= v <= 127 or 128 <= v <= 208 or 240 <= v <= 255
+def _encode_reg(val: Reg) -> int:
+  if isinstance(val, TTMP): return 108 + val.idx
+  return val.idx  # hi bit is handled via opsel, not in register encoding
 
 def encode_src(val) -> int:
   if isinstance(val, VGPR): return 256 + _encode_reg(val)
   if isinstance(val, Reg): return _encode_reg(val)
-  if isinstance(val, SrcMod) and not isinstance(val, Reg): return val.val if _is_inline_const(val.val) else 255
+  if isinstance(val, SrcMod) and not isinstance(val, Reg):
+    # SrcMod wraps either special registers (VCC_LO=106, EXEC_LO=126, etc.) or literals
+    # Special register values are in valid encoding ranges - return as-is
+    # Literals (large integers) need 255 marker
+    v = val.val
+    # Valid source encoding ranges: 0-127 (SGPRs/special), 128-192 (inline const), 193-208 (neg inline), 240-247 (float), 251-253 (special)
+    if 0 <= v <= 127 or 240 <= v <= 255: return v  # SGPRs, special regs, float constants
+    if 128 <= v <= 192: return v  # Inline positive constants (0-64)
+    if 193 <= v <= 208: return v  # Inline negative constants (-1 to -16)
+    return 255  # Literal marker - value stored separately
   if hasattr(val, 'value'): return val.value  # IntEnum
   if isinstance(val, float): return 128 if val == 0.0 else FLOAT_ENC.get(val, 255)
-  if isinstance(val, int): return 128 + val if 0 <= val <= 64 else 192 - val if -16 <= val <= -1 else 255
-  return 255
-
-def decode_src(val: int) -> str:
-  if val <= 105: return f"s{val}"
-  if val in SPECIAL_GPRS: return SPECIAL_GPRS[val]
-  if val in FLOAT_DEC: return FLOAT_DEC[val]
-  if 108 <= val <= 123: return f"ttmp{val - 108}"
-  if 128 <= val <= 192: return str(val - 128)
-  if 193 <= val <= 208: return str(-(val - 192))
-  if 256 <= val <= 511: return f"v{val - 256}"
-  return "lit" if val == 255 else f"?{val}"
+  return 128 + val if isinstance(val, int) and 0 <= val <= 64 else 192 + (-val) if isinstance(val, int) and -16 <= val <= -1 else 255
 
 # Instruction base class
 class Inst:
@@ -274,107 +153,104 @@ class Inst:
     cls._fields = {n: v[0] if isinstance(v, tuple) else v for n, v in cls.__dict__.items() if isinstance(v, BitField) or (isinstance(v, tuple) and len(v) == 2 and isinstance(v[0], BitField))}
     if 'encoding' in cls._fields and isinstance(cls.__dict__.get('encoding'), tuple): cls._encoding = cls.__dict__['encoding']
 
-  def _or_field(self, name: str, bit: int):
-    cur = self._values.get(name, 0)
-    self._values[name] = (cur.val if isinstance(cur, RawImm) else cur) | bit
-
-  def _encode_src(self, name: str, val):
-    """Encode a source field, handling modifiers and literals."""
-    encoded = encode_src(val)
-    has_opsel = 'opsel' in self._fields
-    if isinstance(val, Reg) and val.hi and not has_opsel: encoded |= 0x80  # hi bit in src for VOP1/2/C
-    self._values[name] = RawImm(encoded)
-    # Handle neg/abs/opsel modifiers
-    if isinstance(val, SrcMod):
-      mod_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
-      if val.neg and 'neg' in self._fields: self._or_field('neg', mod_bit)
-      if val.abs_ and 'abs' in self._fields: self._or_field('abs', mod_bit)
-    if isinstance(val, Reg) and val.hi and has_opsel:
-      self._or_field('opsel', {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0))
-    # Track literal value if needed
-    if encoded == 255 and self._literal is None:
-      import struct
-      # Check if THIS source uses 64-bit encoding (not just src0)
-      src_idx = {'src0': 0, 'src1': 1, 'src2': 2, 'ssrc0': 0, 'ssrc1': 1}.get(name, 0)
-      src_regs = self.src_regs(src_idx)
-      is_64 = src_regs == 2
-      if isinstance(val, SrcMod) and not isinstance(val, Reg): lit32 = val.val & MASK32
-      elif isinstance(val, int) and not isinstance(val, IntEnum): lit32 = val & MASK32
-      elif isinstance(val, float): lit32 = (_i64(val) >> 32) if is_64 else _i32(val)  # f64: high 32 bits of f64 repr
-      else: return
-      self._literal = (lit32 << 32) if is_64 else lit32
-
-  def _encode_raw(self, name: str, val):
-    """Encode a raw register field (vdst, vdata, etc.)."""
-    if isinstance(val, Reg):
-      encoded = _encode_reg(val)
-      if val.hi and 'opsel' not in self._fields: encoded |= 0x80
-      self._values[name] = encoded
-      if name == 'vdst' and val.hi and 'opsel' in self._fields: self._or_field('opsel', 8)
-    elif hasattr(val, 'value'): self._values[name] = val.value
-
-  def _validate(self, orig_args: dict):
-    """Format-specific validation. Override in subclass or check by class name."""
-    cls_name, op = self.__class__.__name__, orig_args.get('op')
-    if hasattr(op, 'value'): op = op.value
-    # SMEM: register count must match opcode
-    if cls_name == 'SMEM' and op is not None:
-      expected = {0:1, 1:2, 2:4, 3:8, 4:16, 8:1, 9:2, 10:4, 11:8, 12:16}.get(op)
-      sdata = orig_args.get('sdata')
-      if expected and isinstance(sdata, Reg) and sdata.count != expected:
-        raise ValueError(f"SMEM op {op} expects {expected} registers, got {sdata.count}")
-    # SOP1: b32=1 reg, b64=2 regs
-    if cls_name == 'SOP1' and hasattr(orig_args.get('op'), 'name'):
-      expected = 2 if orig_args['op'].name.endswith('_B64') else 1
-      for fld in ('sdst', 'ssrc0'):
-        if isinstance(orig_args.get(fld), Reg) and orig_args[fld].count != expected:
-          raise ValueError(f"SOP1 {orig_args['op'].name} expects {expected} register(s) for {fld}, got {orig_args[fld].count}")
-
   def __init__(self, *args, literal: int | None = None, **kwargs):
-    self._values, self._literal = dict(self._defaults), None
+    self._values, self._literal = dict(self._defaults), literal
+    # Map positional args to field names
     field_names = [n for n in self._fields if n != 'encoding']
-    orig_args = dict(zip(field_names, args)) | kwargs
+    orig_args = dict(zip(field_names, args))
+    orig_args.update(kwargs)
     self._values.update(orig_args)
-    self._validate(orig_args)
-    # Pre-shift literal for 64-bit sources (literal param is always raw 32-bit value from user)
-    if literal is not None:
-      # Find which source uses the literal (255) and check its register count
-      for n, idx in [('src0', 0), ('src1', 1), ('src2', 2), ('ssrc0', 0), ('ssrc1', 1)]:
-        v = orig_args.get(n)
-        if (isinstance(v, RawImm) and v.val == 255) or (isinstance(v, int) and v == 255):
-          self._literal = (literal << 32) if self.src_regs(idx) == 2 else literal
-          break
-      else:
-        self._literal = literal  # fallback if no literal source found
-    cls_name = self.__class__.__name__
-
-    # Format-specific setup
-    if cls_name == 'FLAT' and 'sve' in self._fields:
-      seg = self._values.get('seg', 0)
-      if (seg.val if isinstance(seg, RawImm) else seg) == 1 and isinstance(orig_args.get('addr'), VGPR): self._values['sve'] = 1
-    if cls_name == 'VOP3P':
-      op = orig_args.get('op')
-      if hasattr(op, 'value'): op = op.value
-      if op in (32, 33, 34) and 'opsel_hi' not in orig_args: self._values['opsel_hi'] = self._values['opsel_hi2'] = 0
-
-    # Encode all fields
+    # Validate register counts for SMEM instructions (before encoding)
+    if self.__class__.__name__ == 'SMEM':
+      op_val = orig_args.get(field_names[0]) if args else orig_args.get('op')
+      if op_val is not None:
+        if hasattr(op_val, 'value'): op_val = op_val.value
+        expected_cnt = {0:1, 1:2, 2:4, 3:8, 4:16, 8:1, 9:2, 10:4, 11:8, 12:16}.get(op_val)
+        sdata_val = orig_args.get('sdata')
+        if expected_cnt is not None and isinstance(sdata_val, Reg) and sdata_val.count != expected_cnt:
+          raise ValueError(f"SMEM op {op_val} expects {expected_cnt} registers, got {sdata_val.count}")
+    # Validate register counts for SOP1 instructions (b32 = 1 reg, b64 = 2 regs)
+    if self.__class__.__name__ == 'SOP1':
+      op_val = orig_args.get(field_names[0]) if args else orig_args.get('op')
+      if op_val is not None and hasattr(op_val, 'name'):
+        expected = 2 if op_val.name.endswith('_B64') else 1
+        sdst_val, ssrc0_val = orig_args.get('sdst'), orig_args.get('ssrc0')
+        if isinstance(sdst_val, Reg) and sdst_val.count != expected:
+          raise ValueError(f"SOP1 {op_val.name} expects {expected} destination register(s), got {sdst_val.count}")
+        if isinstance(ssrc0_val, Reg) and ssrc0_val.count != expected:
+          raise ValueError(f"SOP1 {op_val.name} expects {expected} source register(s), got {ssrc0_val.count}")
+    # Type check and encode values
     for name, val in list(self._values.items()):
       if name == 'encoding': continue
+      # For RawImm, only process RAW_FIELDS to unwrap to int
       if isinstance(val, RawImm):
         if name in RAW_FIELDS: self._values[name] = val.val
         continue
       field = self._fields.get(name)
       marker = field.marker if field else None
       # Type validation
-      if marker is _SGPRField and isinstance(val, VGPR): raise TypeError(f"field '{name}' requires SGPR, got VGPR")
-      if marker is _VGPRField and not isinstance(val, VGPR): raise TypeError(f"field '{name}' requires VGPR, got {type(val).__name__}")
+      if marker is _SGPRField:
+        if isinstance(val, VGPR): raise TypeError(f"field '{name}' requires SGPR, got VGPR")
+        if not isinstance(val, (SGPR, TTMP, SrcMod, int, RawImm)): raise TypeError(f"field '{name}' requires SGPR, got {type(val).__name__}")
+      if marker is _VGPRField:
+        if not isinstance(val, VGPR): raise TypeError(f"field '{name}' requires VGPR, got {type(val).__name__}")
       if marker is _SSrc and isinstance(val, VGPR): raise TypeError(f"field '{name}' requires scalar source, got VGPR")
-      # Encode by field type
-      if name in SRC_FIELDS: self._encode_src(name, val)
-      elif name in RAW_FIELDS: self._encode_raw(name, val)
-      elif name == 'sbase': self._values[name] = (val.idx if isinstance(val, Reg) else val.val if isinstance(val, SrcMod) else val * 2) // 2
-      elif name in {'srsrc', 'ssamp'} and isinstance(val, Reg): self._values[name] = val.idx // 4
-      elif marker is _VDSTYEnc and isinstance(val, VGPR): self._values[name] = val.idx >> 1
+      # Encode source fields as RawImm for consistent disassembly
+      if name in SRC_FIELDS:
+        encoded = encode_src(val)
+        # For VOP1/VOP2/VOPC (no opsel field), encode hi bit in src value
+        if isinstance(val, Reg) and val.hi and 'opsel' not in self._fields:
+          encoded |= 0x80
+        self._values[name] = RawImm(encoded)
+        # Handle neg/abs/opsel modifiers for VOP3 instructions
+        if isinstance(val, SrcMod):
+          if val.neg and 'neg' in self._fields:
+            neg_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+            cur_neg = self._values.get('neg', 0)
+            self._values['neg'] = (cur_neg.val if isinstance(cur_neg, RawImm) else cur_neg) | neg_bit
+          if val.abs_ and 'abs' in self._fields:
+            abs_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+            cur_abs = self._values.get('abs', 0)
+            self._values['abs'] = (cur_abs.val if isinstance(cur_abs, RawImm) else cur_abs) | abs_bit
+        # Handle hi (opsel) for 16-bit ops - only for formats with opsel field
+        if isinstance(val, Reg) and val.hi and 'opsel' in self._fields:
+          opsel_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+          cur_opsel = self._values.get('opsel', 0)
+          self._values['opsel'] = (cur_opsel.val if isinstance(cur_opsel, RawImm) else cur_opsel) | opsel_bit
+        # Track literal value if needed (encoded as 255)
+        # For 64-bit ops, store literal in high 32 bits (to match from_bytes decoding and to_bytes encoding)
+        if encoded == 255 and self._literal is None:
+          if isinstance(val, SrcMod) and not isinstance(val, Reg):
+            # SrcMod wrapping a literal value
+            self._literal = (val.val << 32) if self._is_64bit_op() else val.val
+          elif isinstance(val, int) and not isinstance(val, IntEnum):
+            self._literal = (val << 32) if self._is_64bit_op() else val
+          elif isinstance(val, float):
+            import struct
+            lit32 = struct.unpack('<I', struct.pack('<f', val))[0]
+            self._literal = (lit32 << 32) if self._is_64bit_op() else lit32
+      # Encode raw register fields for consistent repr
+      elif name in RAW_FIELDS:
+        if isinstance(val, Reg):
+          encoded = _encode_reg(val)
+          # For VOP1/VOP2/VOPC (no opsel field), encode hi bit in register value
+          if val.hi and 'opsel' not in self._fields:
+            encoded |= 0x80
+          self._values[name] = encoded
+          # Handle vdst hi (opsel bit 3) for 16-bit ops - only for formats with opsel field
+          if name == 'vdst' and val.hi and 'opsel' in self._fields:
+            cur_opsel = self._values.get('opsel', 0)
+            self._values['opsel'] = (cur_opsel.val if isinstance(cur_opsel, RawImm) else cur_opsel) | 8
+        elif hasattr(val, 'value'): self._values[name] = val.value  # IntEnum like SrcEnum.NULL
+      # Encode sbase (divided by 2) and srsrc/ssamp (divided by 4)
+      elif name == 'sbase':
+        if isinstance(val, Reg): self._values[name] = val.idx // 2
+        elif isinstance(val, SrcMod): self._values[name] = val.val // 2  # Special regs like VCC_LO
+      elif name in {'srsrc', 'ssamp'} and isinstance(val, Reg):
+        self._values[name] = val.idx // 4
+      # VOPD vdsty: encode as actual >> 1 (constraint: vdsty parity must be opposite of vdstx)
+      elif marker is _VDSTYEnc and isinstance(val, VGPR):
+        self._values[name] = val.idx >> 1
 
   def _encode_field(self, name: str, val) -> int:
     if isinstance(val, RawImm): return val.val
@@ -397,36 +273,28 @@ class Inst:
     return None
 
   def _is_64bit_op(self) -> bool:
-    """Check if this instruction uses 64-bit operands (and thus 64-bit literals)."""
+    """Check if this instruction uses 64-bit operands (and thus 64-bit literals).
+    Exception: V_LDEXP_F64 has 32-bit integer src1, so its literal is 32-bit."""
     op = self._values.get('op')
     if op is None: return False
+    # op may be an enum (from __init__) or an int (from from_int)
     op_name = op.name if hasattr(op, 'name') else None
-    # Look up op name from int if needed (happens in from_bytes path)
     if op_name is None and self.__class__.__name__ == 'VOP3':
+      from extra.assembly.amd.autogen.rdna3 import VOP3Op
       try: op_name = VOP3Op(op).name
       except ValueError: pass
-    if op_name is None and self.__class__.__name__ == 'VOPC':
-      try: op_name = VOPCOp(op).name
-      except ValueError: pass
     if op_name is None: return False
-    # V_LDEXP_F64 has 32-bit integer src1, so literal is 32-bit
-    return op_name != 'V_LDEXP_F64' and op_name.endswith(('_F64', '_B64', '_I64', '_U64'))
+    # V_LDEXP_F64 has 32-bit integer exponent in src1, so literal is 32-bit
+    if op_name == 'V_LDEXP_F64': return False
+    return op_name.endswith(('_F64', '_B64', '_I64', '_U64'))
 
   def to_bytes(self) -> bytes:
     result = self.to_int().to_bytes(self._size(), 'little')
     lit = self._get_literal() or getattr(self, '_literal', None)
     if lit is None: return result
-    # For 64-bit sources, literal is stored in high 32 bits internally, but encoded as 4 bytes
-    # Find which source uses the literal (255) and check its register count
-    lit_src_is_64 = False
-    for n, idx in [('src0', 0), ('src1', 1), ('src2', 2), ('ssrc0', 0), ('ssrc1', 1)]:
-      if n not in self._values: continue
-      v = self._values[n]
-      if (isinstance(v, RawImm) and v.val == 255) or (isinstance(v, int) and v == 255):
-        lit_src_is_64 = self.is_src_64(idx)
-        break
-    lit32 = (lit >> 32) if lit_src_is_64 else lit
-    return result + (lit32 & MASK32).to_bytes(4, 'little')
+    # For 64-bit ops, literal is stored in high 32 bits internally, but encoded as 4 bytes
+    lit32 = (lit >> 32) if self._is_64bit_op() else lit
+    return result + (lit32 & 0xffffffff).to_bytes(4, 'little')
 
   @classmethod
   def _size(cls) -> int: return 4 if issubclass(cls, Inst32) else 8
@@ -447,24 +315,14 @@ class Inst:
     op_val = inst._values.get('op', 0)
     has_literal = cls.__name__ == 'VOP2' and op_val in (44, 45, 55, 56)
     has_literal = has_literal or (cls.__name__ == 'SOP2' and op_val in (69, 70))
-    # VOPD fmaak/fmamk always have a literal (opx/opy value 1 or 2)
-    opx, opy = inst._values.get('opx', 0), inst._values.get('opy', 0)
-    has_literal = has_literal or (cls.__name__ == 'VOPD' and (opx in (1, 2) or opy in (1, 2)))
     for n in SRC_FIELDS:
       if n in inst._values and isinstance(inst._values[n], RawImm) and inst._values[n].val == 255: has_literal = True
     if has_literal:
       # For 64-bit ops, the literal is 32 bits placed in the HIGH 32 bits of the 64-bit value
       # (low 32 bits are zero). This is how AMD hardware interprets 32-bit literals for 64-bit ops.
-      # Check which source uses the literal and whether THAT source is 64-bit
       if len(data) >= cls._size() + 4:
         lit32 = int.from_bytes(data[cls._size():cls._size()+4], 'little')
-        # Find which source has literal (255) and check its register count
-        lit_src_is_64 = False
-        for n, idx in [('src0', 0), ('src1', 1), ('src2', 2)]:
-          if n in inst._values and isinstance(inst._values[n], RawImm) and inst._values[n].val == 255:
-            lit_src_is_64 = inst.src_regs(idx) == 2
-            break
-        inst._literal = (lit32 << 32) if lit_src_is_64 else lit32
+        inst._literal = (lit32 << 32) if inst._is_64bit_op() else lit32
     return inst
 
   def __repr__(self):
@@ -475,14 +333,6 @@ class Inst:
     lit = f", literal={hex(self._literal)}" if self._literal is not None else ""
     return f"{self.__class__.__name__}({', '.join(f'{k}={v}' for k, v in items)}{lit})"
 
-  def __getattr__(self, name: str):
-    if name.startswith('_'): raise AttributeError(name)
-    return unwrap(self._values.get(name, 0))
-
-  def lit(self, v: int, neg: bool = False) -> str:
-    s = f"0x{self._literal:x}" if v == 255 and self._literal else decode_src(v)
-    return f"-{s}" if neg else s
-
   def __eq__(self, other):
     if not isinstance(other, Inst): return NotImplemented
     return self.__class__ == other.__class__ and self._values == other._values and self._literal == other._literal
@@ -493,41 +343,298 @@ class Inst:
     from extra.assembly.amd.asm import disasm
     return disasm(self)
 
-  _enum_map = {'VOP1': VOP1Op, 'VOP2': VOP2Op, 'VOP3': VOP3Op, 'VOP3SD': VOP3SDOp, 'VOP3P': VOP3POp, 'VOPC': VOPCOp,
-               'SOP1': SOP1Op, 'SOP2': SOP2Op, 'SOPC': SOPCOp, 'SOPK': SOPKOp, 'SOPP': SOPPOp,
-               'SMEM': SMEMOp, 'DS': DSOp, 'FLAT': FLATOp, 'MUBUF': MUBUFOp, 'MTBUF': MTBUFOp, 'MIMG': MIMGOp,
-               'VOPD': VOPDOp, 'VINTERP': VINTERPOp}
-  _VOP3SD_OPS = {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}
-
-  @property
-  def op(self):
-    """Return the op as an enum (e.g., VOP1Op.V_MOV_B32). VOP3 returns VOPCOp/VOP3SDOp for those op ranges."""
-    val = self._values.get('op')
-    if val is None: return None
-    if hasattr(val, 'name'): return val  # already an enum
-    cls_name = self.__class__.__name__
-    assert cls_name in self._enum_map, f"no enum map for {cls_name}"
-    return self._enum_map[cls_name](val)
-
-  @cached_property
-  def op_name(self) -> str:
-    op = self.op
-    return op.name if hasattr(op, 'name') else ''
-
-  @cached_property
-  def _spec_regs(self) -> tuple[int, int, int, int]: return spec_regs(self.op_name)
-  @cached_property
-  def _spec_dtype(self) -> tuple[str | None, str | None, str | None, str | None]: return spec_dtype(self.op_name)
-  def dst_regs(self) -> int: return self._spec_regs[0]
-  def src_regs(self, n: int) -> int: return self._spec_regs[n + 1]
-  def num_srcs(self) -> int: return spec_num_srcs(self.op_name)
-  def dst_dtype(self) -> str | None: return self._spec_dtype[0]
-  def src_dtype(self, n: int) -> str | None: return self._spec_dtype[n + 1]
-  def is_src_16(self, n: int) -> bool: return self._spec_regs[n + 1] == 1 and is_dtype_16(self._spec_dtype[n + 1])
-  def is_src_64(self, n: int) -> bool: return self._spec_regs[n + 1] == 2
-  def is_16bit(self) -> bool: return spec_is_16bit(self.op_name)
-  def is_64bit(self) -> bool: return spec_is_64bit(self.op_name)
-  def is_dst_16(self) -> bool: return self._spec_regs[0] == 1 and is_dtype_16(self._spec_dtype[0])
-
 class Inst32(Inst): pass
 class Inst64(Inst): pass
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CODE GENERATION: generates autogen/__init__.py by parsing AMD ISA PDFs
+# Supports both RDNA3.5 and CDNA4 instruction set PDFs - auto-detects format
+# ═══════════════════════════════════════════════════════════════════════════════
+
+PDF_URLS = {
+  "rdna3": "https://docs.amd.com/api/khub/documents/UVVZM22UN7tMUeiW_4ShTQ/content",  # RDNA3.5
+  "rdna4": "https://docs.amd.com/api/khub/documents/uQpkEvk3pv~kfAb2x~j4uw/content",
+  "cdna": ["https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-mi300-cdna3-instruction-set-architecture.pdf",
+           "https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf"],
+}
+FIELD_TYPES = {'SSRC0': 'SSrc', 'SSRC1': 'SSrc', 'SOFFSET': 'SSrc', 'SADDR': 'SSrc', 'SRC0': 'Src', 'SRC1': 'Src', 'SRC2': 'Src',
+  'SDST': 'SGPRField', 'SBASE': 'SGPRField', 'SDATA': 'SGPRField', 'SRSRC': 'SGPRField', 'VDST': 'VGPRField', 'VSRC1': 'VGPRField', 'VDATA': 'VGPRField',
+  'VADDR': 'VGPRField', 'ADDR': 'VGPRField', 'DATA': 'VGPRField', 'DATA0': 'VGPRField', 'DATA1': 'VGPRField', 'SIMM16': 'SImm', 'OFFSET': 'Imm',
+  'OPX': 'VOPDOp', 'OPY': 'VOPDOp', 'SRCX0': 'Src', 'SRCY0': 'Src', 'VSRCX1': 'VGPRField', 'VSRCY1': 'VGPRField', 'VDSTX': 'VGPRField', 'VDSTY': 'VDSTYEnc'}
+FIELD_ORDER = {
+  'SOP2': ['op', 'sdst', 'ssrc0', 'ssrc1'], 'SOP1': ['op', 'sdst', 'ssrc0'], 'SOPC': ['op', 'ssrc0', 'ssrc1'],
+  'SOPK': ['op', 'sdst', 'simm16'], 'SOPP': ['op', 'simm16'], 'VOP1': ['op', 'vdst', 'src0'], 'VOPC': ['op', 'src0', 'vsrc1'],
+  'VOP2': ['op', 'vdst', 'src0', 'vsrc1'], 'VOP3SD': ['op', 'vdst', 'sdst', 'src0', 'src1', 'src2', 'clmp'],
+  'SMEM': ['op', 'sdata', 'sbase', 'soffset', 'offset', 'glc', 'dlc'], 'DS': ['op', 'vdst', 'addr', 'data0', 'data1'],
+  'VOP3': ['op', 'vdst', 'src0', 'src1', 'src2', 'omod', 'neg', 'abs', 'clmp', 'opsel'],
+  'VOP3P': ['op', 'vdst', 'src0', 'src1', 'src2', 'neg', 'neg_hi', 'opsel', 'opsel_hi', 'clmp'],
+  'FLAT': ['op', 'vdst', 'addr', 'data', 'saddr', 'offset', 'seg', 'dlc', 'glc', 'slc'],
+  'MUBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
+  'MTBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'format', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
+  'MIMG': ['op', 'vdata', 'vaddr', 'srsrc', 'ssamp', 'dmask', 'dim', 'unrm', 'dlc', 'glc', 'slc'],
+  'EXP': ['en', 'target', 'vsrc0', 'vsrc1', 'vsrc2', 'vsrc3', 'done', 'row'],
+  'VINTERP': ['op', 'vdst', 'src0', 'src1', 'src2', 'waitexp', 'clmp', 'opsel', 'neg'],
+  'VOPD': ['opx', 'opy', 'vdstx', 'vdsty', 'srcx0', 'vsrcx1', 'srcy0', 'vsrcy1'],
+  'LDSDIR': ['op', 'vdst', 'attr', 'attr_chan', 'wait_va']}
+SRC_EXTRAS = {233: 'DPP8', 234: 'DPP8FI', 250: 'DPP16', 251: 'VCCZ', 252: 'EXECZ', 254: 'LDS_DIRECT'}
+FLOAT_MAP = {'0.5': 'POS_HALF', '-0.5': 'NEG_HALF', '1.0': 'POS_ONE', '-1.0': 'NEG_ONE', '2.0': 'POS_TWO', '-2.0': 'NEG_TWO',
+  '4.0': 'POS_FOUR', '-4.0': 'NEG_FOUR', '1/(2*PI)': 'INV_2PI', '0': 'ZERO'}
+
+def _parse_bits(s: str) -> tuple[int, int] | None:
+  import re
+  return (int(m.group(1)), int(m.group(2) or m.group(1))) if (m := re.match(r'\[(\d+)(?::(\d+))?\]', s)) else None
+
+def _parse_fields_table(table: list, fmt: str, enums: set[str]) -> list[tuple]:
+  import re
+  fields = []
+  for row in table[1:]:
+    if not row or not row[0]: continue
+    name, bits_str = row[0].split('\n')[0].strip(), (row[1] or '').split('\n')[0].strip()
+    if not (bits := _parse_bits(bits_str)): continue
+    enc_val, hi, lo = None, bits[0], bits[1]
+    if name == 'ENCODING' and row[2]:
+      # Handle both RDNA3 ('bXX) and CDNA4 (Must be: XX) encoding formats
+      if m := re.search(r"(?:'b|Must be:\s*)([01_]+)", row[2]):
+        enc_bits = m.group(1).replace('_', '')
+        enc_val = int(enc_bits, 2)
+        declared_width, actual_width = hi - lo + 1, len(enc_bits)
+        if actual_width > declared_width: lo = hi - actual_width + 1
+    ftype = f"{fmt}Op" if name == 'OP' and f"{fmt}Op" in enums else FIELD_TYPES.get(name.upper())
+    fields.append((name, hi, lo, enc_val, ftype))
+  return fields
+
+def _parse_single_pdf(url: str) -> dict:
+  """Parse a single PDF and return raw data (formats, enums, src_enum, doc_name, is_cdna)."""
+  import re, pdfplumber
+  from tinygrad.helpers import fetch
+
+  pdf = pdfplumber.open(fetch(url))
+
+  # Auto-detect document type from first page
+  first_page_text = pdf.pages[0].extract_text() or ''
+  is_cdna4 = 'CDNA4' in first_page_text or 'CDNA 4' in first_page_text
+  is_cdna3 = 'CDNA3' in first_page_text or 'CDNA 3' in first_page_text or 'MI300' in first_page_text
+  is_cdna = is_cdna3 or is_cdna4
+  is_rdna4 = 'RDNA4' in first_page_text or 'RDNA 4' in first_page_text
+  is_rdna35 = 'RDNA3.5' in first_page_text or 'RDNA 3.5' in first_page_text  # Check 3.5 before 3
+  is_rdna3 = not is_rdna35 and ('RDNA3' in first_page_text or 'RDNA 3' in first_page_text)
+  doc_name = "CDNA4" if is_cdna4 else "CDNA3" if is_cdna3 else "RDNA4" if is_rdna4 else "RDNA3.5" if is_rdna35 else "RDNA3" if is_rdna3 else "Unknown"
+
+  # Find the "Microcode Formats" section - search for SOP2 format definition
+  microcode_start = None
+  total_pages = len(pdf.pages)
+  # Search from likely locations (formats are typically 20-95% through the document - RDNA3 has them at ~25%)
+  for i in range(int(total_pages * 0.2), total_pages):
+    text = pdf.pages[i].extract_text() or ''
+    # Look for "X.Y.Z. SOP2" section header or "Chapter X. Microcode Formats"
+    if re.search(r'\d+\.\d+\.\d+\.\s+SOP2\b', text) or re.search(r'Chapter \d+\.\s+Microcode Formats', text):
+      microcode_start = i
+      break
+  if microcode_start is None: microcode_start = int(total_pages * 0.9)
+
+  pages = pdf.pages[microcode_start:microcode_start + 50]
+  page_texts = [p.extract_text() or '' for p in pages]
+  page_tables = [[t.extract() for t in p.find_tables()] for p in pages]
+  full_text = '\n'.join(page_texts)
+
+  # parse SSRC encoding from first page with VCC_LO
+  src_enum = dict(SRC_EXTRAS)
+  for text in page_texts[:10]:
+    if 'SSRC0' in text and 'VCC_LO' in text:
+      for m in re.finditer(r'^(\d+)\s+(\S+)', text, re.M):
+        val, name = int(m.group(1)), m.group(2).rstrip('.:')
+        if name in FLOAT_MAP: src_enum[val] = FLOAT_MAP[name]
+        elif re.match(r'^[A-Z][A-Z0-9_]*$', name): src_enum[val] = name
+      break
+
+  # parse opcode tables
+  enums: dict[str, dict[int, str]] = {}
+  for m in re.finditer(r'Table \d+\. (\w+) Opcodes(.*?)(?=Table \d+\.|\n\d+\.\d+\.\d+\.\s+\w+\s*\nDescription|$)', full_text, re.S):
+    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+([A-Z][A-Z0-9_]+)', m.group(2))}:
+      enums[m.group(1) + "Op"] = ops
+  if vopd_m := re.search(r'Table \d+\. VOPD Y-Opcodes\n(.*?)(?=Table \d+\.|15\.\d)', full_text, re.S):
+    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+(V_DUAL_\w+)', vopd_m.group(1))}:
+      enums["VOPDOp"] = ops
+  enum_names = set(enums.keys())
+
+  def is_fields_table(t) -> bool: return t and len(t) > 1 and t[0] and 'Field' in str(t[0][0] or '')
+  def has_encoding(fields) -> bool: return any(f[0] == 'ENCODING' for f in fields)
+  def has_header_before_fields(text) -> bool:
+    return (pos := text.find('Field Name')) != -1 and bool(re.search(r'\d+\.\d+\.\d+\.\s+\w+\s*\n', text[:pos]))
+
+  # find format headers with their page indices
+  format_headers = []
+  for i, text in enumerate(page_texts):
+    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n?Description', text): format_headers.append((m.group(1), i, m.start()))
+    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n', text):
+      fmt_name = m.group(1)
+      if is_cdna and fmt_name.isupper() and len(fmt_name) >= 2:
+        format_headers.append((fmt_name, i, m.start()))
+      elif m.start() > len(text) - 200 and 'Description' not in text[m.end():] and i + 1 < len(page_texts):
+        next_text = page_texts[i + 1].lstrip()
+        if next_text.startswith('Description') or (next_text.startswith('"RDNA') and 'Description' in next_text[:200]):
+          format_headers.append((fmt_name, i, m.start()))
+
+  # parse instruction formats
+  formats: dict[str, list] = {}
+  for fmt_name, page_idx, header_pos in format_headers:
+    if fmt_name in formats: continue
+    text, tables = page_texts[page_idx], page_tables[page_idx]
+    field_pos = text.find('Field Name', header_pos)
+
+    fields = None
+    for offset in range(3):
+      if page_idx + offset >= len(pages): break
+      if offset > 0 and has_header_before_fields(page_texts[page_idx + offset]): break
+      for t in page_tables[page_idx + offset] if offset > 0 or field_pos > header_pos else []:
+        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)) and has_encoding(f):
+          fields = f
+          break
+      if fields: break
+
+    if not fields and field_pos > header_pos:
+      for t in tables:
+        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)):
+          fields = f
+          break
+
+    if not fields: continue
+    field_names = {f[0] for f in fields}
+
+    for pg_offset in range(1, 3):
+      if page_idx + pg_offset >= len(pages) or has_header_before_fields(page_texts[page_idx + pg_offset]): break
+      for t in page_tables[page_idx + pg_offset]:
+        if is_fields_table(t) and (extra := _parse_fields_table(t, fmt_name, enum_names)) and not has_encoding(extra):
+          for ef in extra:
+            if ef[0] not in field_names:
+              fields.append(ef)
+              field_names.add(ef[0])
+          break
+    formats[fmt_name] = fields
+
+  # fix known PDF errors
+  if 'SMEM' in formats:
+    formats['SMEM'] = [(n, 13 if n == 'DLC' else 14 if n == 'GLC' else h, 13 if n == 'DLC' else 14 if n == 'GLC' else l, e, t)
+                       for n, h, l, e, t in formats['SMEM']]
+
+  return {"formats": formats, "enums": enums, "src_enum": src_enum, "doc_name": doc_name, "is_cdna": is_cdna}
+
+def _merge_results(results: list[dict]) -> dict:
+  """Merge multiple PDF parse results into a superset. Asserts if any conflicts."""
+  merged = {"formats": {}, "enums": {}, "src_enum": dict(SRC_EXTRAS), "doc_names": [], "is_cdna": False}
+  for r in results:
+    merged["doc_names"].append(r["doc_name"])
+    merged["is_cdna"] = merged["is_cdna"] or r["is_cdna"]
+    # Merge src_enum (union, assert no conflicts)
+    for val, name in r["src_enum"].items():
+      if val in merged["src_enum"]:
+        assert merged["src_enum"][val] == name, f"SrcEnum conflict: {val} = {merged['src_enum'][val]} vs {name}"
+      else:
+        merged["src_enum"][val] = name
+    # Merge enums (union of ops per enum, assert no conflicts)
+    for enum_name, ops in r["enums"].items():
+      if enum_name not in merged["enums"]: merged["enums"][enum_name] = {}
+      for val, name in ops.items():
+        if val in merged["enums"][enum_name]:
+          assert merged["enums"][enum_name][val] == name, f"{enum_name} conflict: {val} = {merged['enums'][enum_name][val]} vs {name}"
+        else:
+          merged["enums"][enum_name][val] = name
+    # Merge formats (union of fields, assert no bit position conflicts for same field name)
+    for fmt_name, fields in r["formats"].items():
+      if fmt_name not in merged["formats"]:
+        merged["formats"][fmt_name] = list(fields)
+      else:
+        existing = {f[0]: (f[1], f[2]) for f in merged["formats"][fmt_name]}  # name -> (hi, lo)
+        for f in fields:
+          name, hi, lo = f[0], f[1], f[2]
+          if name in existing:
+            assert existing[name] == (hi, lo), f"Format {fmt_name} field {name} conflict: bits {existing[name]} vs ({hi}, {lo})"
+          else:
+            merged["formats"][fmt_name].append(f)
+  return merged
+
+def generate(output_path: str | None = None, arch: str = "rdna3") -> dict:
+  """Generate instruction definitions from AMD ISA PDF(s). Returns dict with formats for testing."""
+  urls = PDF_URLS[arch]
+  if isinstance(urls, str): urls = [urls]
+
+  # Parse all PDFs and merge
+  results = [_parse_single_pdf(url) for url in urls]
+  if len(results) == 1:
+    merged = results[0]
+    doc_name = merged["doc_name"]
+  else:
+    merged = _merge_results(results)
+    doc_name = "+".join(merged["doc_names"])
+
+  formats, enums, src_enum = merged["formats"], merged["enums"], merged["src_enum"]
+
+  # generate output
+  def enum_lines(name, items):
+    return [f"class {name}(IntEnum):"] + [f"  {n} = {v}" for v, n in sorted(items.items())] + [""]
+  def field_key(f): return order.index(f[0].lower()) if f[0].lower() in order else 1000
+  lines = [f"# autogenerated from AMD {doc_name} ISA PDF by dsl.py - do not edit", "from enum import IntEnum",
+           "from typing import Annotated",
+           "from extra.assembly.amd.dsl import bits, BitField, Inst32, Inst64, SGPR, VGPR, TTMP as TTMP, s as s, v as v, ttmp as ttmp, SSrc, Src, SImm, Imm, VDSTYEnc, SGPRField, VGPRField",
+           "import functools", ""]
+  lines += enum_lines("SrcEnum", src_enum) + sum([enum_lines(n, ops) for n, ops in sorted(enums.items())], [])
+  # Format-specific field defaults (verified against LLVM test vectors)
+  format_defaults = {'VOP3P': {'opsel_hi': 3, 'opsel_hi2': 1}}
+  lines.append("# instruction formats")
+  for fmt_name, fields in sorted(formats.items()):
+    base = "Inst64" if max(f[1] for f in fields) > 31 or fmt_name == 'VOP3SD' else "Inst32"
+    order = FIELD_ORDER.get(fmt_name, [])
+    lines.append(f"class {fmt_name}({base}):")
+    if enc := next((f for f in fields if f[0] == 'ENCODING'), None):
+      enc_str = f"bits[{enc[1]}:{enc[2]}] == 0b{enc[3]:b}" if enc[1] != enc[2] else f"bits[{enc[1]}] == {enc[3]}"
+      lines.append(f"  encoding = {enc_str}")
+    if defaults := format_defaults.get(fmt_name):
+      lines.append(f"  _defaults = {defaults}")
+    for name, hi, lo, _, ftype in sorted([f for f in fields if f[0] != 'ENCODING'], key=field_key):
+      if ftype and ftype.endswith('Op'):
+        ann = f":Annotated[BitField, {ftype}]"
+      else:
+        ann = f":{ftype}" if ftype else ""
+      lines.append(f"  {name.lower()}{ann} = bits[{hi}]" if hi == lo else f"  {name.lower()}{ann} = bits[{hi}:{lo}]")
+    lines.append("")
+  lines.append("# instruction helpers")
+  for cls_name, ops in sorted(enums.items()):
+    fmt = cls_name[:-2]
+    for op_val, name in sorted(ops.items()):
+      seg = {"GLOBAL": ", seg=2", "SCRATCH": ", seg=2"}.get(fmt, "")
+      tgt = {"GLOBAL": "FLAT, GLOBALOp", "SCRATCH": "FLAT, SCRATCHOp"}.get(fmt, f"{fmt}, {cls_name}")
+      if fmt in formats or fmt in ("GLOBAL", "SCRATCH"):
+        if fmt in ("VOP1", "VOP2", "VOPC"):
+          suffix = "_e32"
+        elif fmt == "VOP3" and op_val < 512:
+          suffix = "_e64"
+        else:
+          suffix = ""
+        if name in ('V_FMAMK_F32', 'V_FMAMK_F16'):
+          lines.append(f"def {name.lower()}{suffix}(vdst, src0, K, vsrc1): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
+        elif name in ('V_FMAAK_F32', 'V_FMAAK_F16'):
+          lines.append(f"def {name.lower()}{suffix}(vdst, src0, vsrc1, K): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
+        else:
+          lines.append(f"{name.lower()}{suffix} = functools.partial({tgt}.{name}{seg})")
+  skip_exports = {'DPP8', 'DPP16'}
+  src_names = {name for _, name in src_enum.items()}
+  lines += [""] + [f"{name} = SrcEnum.{name}" for _, name in sorted(src_enum.items()) if name not in skip_exports]
+  if "NULL" in src_names: lines.append("OFF = NULL\n")
+
+  if output_path is not None:
+    import pathlib
+    pathlib.Path(output_path).write_text('\n'.join(lines))
+  return {"formats": formats, "enums": enums, "src_enum": src_enum}
+
+if __name__ == "__main__":
+  import argparse
+  parser = argparse.ArgumentParser(description="Generate instruction definitions from AMD ISA PDF")
+  parser.add_argument("--arch", choices=list(PDF_URLS.keys()) + ["all"], default="rdna3", help="Target architecture (default: rdna3)")
+  args = parser.parse_args()
+  if args.arch == "all":
+    for arch in PDF_URLS.keys():
+      result = generate(f"extra/assembly/amd/autogen/{arch}/__init__.py", arch=arch)
+      print(f"{arch}: generated SrcEnum ({len(result['src_enum'])}) + {len(result['enums'])} opcode enums + {len(result['formats'])} format classes")
+  else:
+    result = generate(f"extra/assembly/amd/autogen/{args.arch}/__init__.py", arch=args.arch)
+    print(f"generated SrcEnum ({len(result['src_enum'])}) + {len(result['enums'])} opcode enums + {len(result['formats'])} format classes")

extra/assembly/amd/emu.py

@@ -1,39 +1,55 @@
 # RDNA3 emulator - executes compiled pseudocode from AMD ISA PDF
 # mypy: ignore-errors
 from __future__ import annotations
-import ctypes
-from extra.assembly.amd.dsl import Inst, unwrap, FLOAT_ENC, MASK32, MASK64, _f32, _i32, _sext, _f16, _i16, _f64, _i64
-from extra.assembly.amd.pcode import Reg
-from extra.assembly.amd.asm import detect_format
+import ctypes, os
+from extra.assembly.amd.dsl import Inst, RawImm
+from extra.assembly.amd.pcode import _f32, _i32, _sext, _f16, _i16, _f64, _i64, Reg
 from extra.assembly.amd.autogen.rdna3.gen_pcode import get_compiled_functions
-from extra.assembly.amd.autogen.rdna3.ins import (SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, VOP1, VOP2, VOP3, VOP3SD, VOP3P, VOPC, DS, FLAT, VOPD,
-  SrcEnum, SOPPOp, SMEMOp, VOP1Op, VOP2Op, VOP3Op, VOP3SDOp, VOP3POp, VOPCOp, GLOBALOp, FLATOp, DSOp, VOPDOp)
+from extra.assembly.amd.autogen.rdna3 import (
+  SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, VOP1, VOP2, VOP3, VOP3SD, VOP3P, VOPC, DS, FLAT, VOPD, SrcEnum,
+  SOP1Op, SOP2Op, SOPCOp, SOPKOp, SOPPOp, SMEMOp, VOP1Op, VOP2Op, VOP3Op, VOP3SDOp, VOP3POp, VOPCOp, DSOp, FLATOp, GLOBALOp, VOPDOp
+)
 
 Program = dict[int, Inst]
 WAVE_SIZE, SGPR_COUNT, VGPR_COUNT = 32, 128, 256
 VCC_LO, VCC_HI, NULL, EXEC_LO, EXEC_HI, SCC = SrcEnum.VCC_LO, SrcEnum.VCC_HI, SrcEnum.NULL, SrcEnum.EXEC_LO, SrcEnum.EXEC_HI, SrcEnum.SCC
 
-# Inline constants for src operands 128-254. Build tables for f32, f16, and f64 formats.
-_FLOAT_CONSTS = {v: k for k, v in FLOAT_ENC.items()} | {248: 0.15915494309189535}  # INV_2PI
-def _build_inline_consts(mask, to_bits):
-  tbl = list(range(65)) + [((-i) & mask) for i in range(1, 17)] + [0] * (127 - 81)
-  for k, v in _FLOAT_CONSTS.items(): tbl[k - 128] = to_bits(v)
-  return tbl
-_INLINE_CONSTS = _build_inline_consts(MASK32, _i32)
-_INLINE_CONSTS_F16 = _build_inline_consts(0xffff, _i16)
-_INLINE_CONSTS_F64 = _build_inline_consts(MASK64, _i64)
+# VOP3 ops that use 64-bit operands (and thus 64-bit literals when src is 255)
+# Exception: V_LDEXP_F64 has 32-bit integer src1, so literal should NOT be 64-bit when src1=255
+_VOP3_64BIT_OPS = {op.value for op in VOP3Op if op.name.endswith(('_F64', '_B64', '_I64', '_U64'))}
+# Ops where src1 is 32-bit (exponent/shift amount) even though the op name suggests 64-bit
+_VOP3_64BIT_OPS_32BIT_SRC1 = {VOP3Op.V_LDEXP_F64.value}
+# Ops with 16-bit types in name (for source/dest handling)
+# Exception: SAD/MSAD ops take 32-bit packed sources and extract 16-bit/8-bit chunks internally
+_VOP3_16BIT_OPS = {op for op in VOP3Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16')) and 'SAD' not in op.name}
+_VOP1_16BIT_OPS = {op for op in VOP1Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
+_VOP2_16BIT_OPS = {op for op in VOP2Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
+# CVT ops with 32/64-bit source (despite 16-bit in name)
+_CVT_32_64_SRC_OPS = {op for op in VOP3Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))} | \
+                     {op for op in VOP1Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))}
+# 16-bit dst ops (PACK has 32-bit dst despite F16 in name)
+_VOP3_16BIT_DST_OPS = {op for op in _VOP3_16BIT_OPS if 'PACK' not in op.name}
+_VOP1_16BIT_DST_OPS = {op for op in _VOP1_16BIT_OPS if 'PACK' not in op.name}
 
-# Helper: extract/write 16-bit half from/to 32-bit value
-def _src16(raw: int, is_hi: bool) -> int: return ((raw >> 16) & 0xffff) if is_hi else (raw & 0xffff)
-def _dst16(cur: int, val: int, is_hi: bool) -> int: return (cur & 0x0000ffff) | ((val & 0xffff) << 16) if is_hi else (cur & 0xffff0000) | (val & 0xffff)
-def _vgpr_hi(src: int) -> bool: return src >= 256 and ((src - 256) & 0x80) != 0
-def _vgpr_masked(src: int) -> int: return ((src - 256) & 0x7f) + 256 if src >= 256 else src
+# Inline constants for src operands 128-254. Build tables for f32, f16, and f64 formats.
+import struct as _struct
+_FLOAT_CONSTS = {SrcEnum.POS_HALF: 0.5, SrcEnum.NEG_HALF: -0.5, SrcEnum.POS_ONE: 1.0, SrcEnum.NEG_ONE: -1.0,
+                 SrcEnum.POS_TWO: 2.0, SrcEnum.NEG_TWO: -2.0, SrcEnum.POS_FOUR: 4.0, SrcEnum.NEG_FOUR: -4.0, SrcEnum.INV_2PI: 0.15915494309189535}
+def _build_inline_consts(neg_mask, float_to_bits):
+  tbl = list(range(65)) + [((-i) & neg_mask) for i in range(1, 17)] + [0] * (127 - 81)
+  for k, v in _FLOAT_CONSTS.items(): tbl[k - 128] = float_to_bits(v)
+  return tbl
+_INLINE_CONSTS = _build_inline_consts(0xffffffff, lambda f: _struct.unpack('<I', _struct.pack('<f', f))[0])
+_INLINE_CONSTS_F16 = _build_inline_consts(0xffff, lambda f: _struct.unpack('<H', _struct.pack('<e', f))[0])
+_INLINE_CONSTS_F64 = _build_inline_consts(0xffffffffffffffff, lambda f: _struct.unpack('<Q', _struct.pack('<d', f))[0])
 
 # Memory access
 _valid_mem_ranges: list[tuple[int, int]] = []
 def set_valid_mem_ranges(ranges: set[tuple[int, int]]) -> None: _valid_mem_ranges.clear(); _valid_mem_ranges.extend(ranges)
 def _mem_valid(addr: int, size: int) -> bool:
-  return not _valid_mem_ranges or any(s <= addr and addr + size <= s + z for s, z in _valid_mem_ranges)
+  for s, z in _valid_mem_ranges:
+    if s <= addr and addr + size <= s + z: return True
+  return not _valid_mem_ranges
 def _ctypes_at(addr: int, size: int): return (ctypes.c_uint8 if size == 1 else ctypes.c_uint16 if size == 2 else ctypes.c_uint32).from_address(addr)
 def mem_read(addr: int, size: int) -> int: return _ctypes_at(addr, size).value if _mem_valid(addr, size) else 0
 def mem_write(addr: int, size: int, val: int) -> None:
@@ -51,6 +67,8 @@ _D16_LOAD_MAP = {'LOAD_D16_U8': (1,0,0), 'LOAD_D16_I8': (1,1,0), 'LOAD_D16_B16':
 _D16_STORE_MAP = {'STORE_D16_HI_B8': (1,1), 'STORE_D16_HI_B16': (2,1)}  # (size, hi)
 FLAT_D16_LOAD = _mem_ops([GLOBALOp, FLATOp], _D16_LOAD_MAP)
 FLAT_D16_STORE = _mem_ops([GLOBALOp, FLATOp], _D16_STORE_MAP)
+DS_LOAD = {DSOp.DS_LOAD_B32: (1,4,0), DSOp.DS_LOAD_B64: (2,4,0), DSOp.DS_LOAD_B128: (4,4,0), DSOp.DS_LOAD_U8: (1,1,0), DSOp.DS_LOAD_I8: (1,1,1), DSOp.DS_LOAD_U16: (1,2,0), DSOp.DS_LOAD_I16: (1,2,1)}
+DS_STORE = {DSOp.DS_STORE_B32: (1,4), DSOp.DS_STORE_B64: (2,4), DSOp.DS_STORE_B128: (4,4), DSOp.DS_STORE_B8: (1,1), DSOp.DS_STORE_B16: (1,2)}
 SMEM_LOAD = {SMEMOp.S_LOAD_B32: 1, SMEMOp.S_LOAD_B64: 2, SMEMOp.S_LOAD_B128: 4, SMEMOp.S_LOAD_B256: 8, SMEMOp.S_LOAD_B512: 16}
 
 # VOPD op -> VOP3 op mapping (VOPD is dual-issue of VOP1/VOP2 ops, use VOP3 enums for pseudocode lookup)
@@ -72,59 +90,132 @@ def _get_compiled() -> dict:
   return _COMPILED
 
 class WaveState:
-  __slots__ = ('sgpr', 'vgpr', 'scc', 'pc', 'literal', '_pend_sgpr')
+  __slots__ = ('sgpr', 'vgpr', 'scc', 'pc', 'literal', '_pend_sgpr', '_scc_reg', '_vcc_reg', '_exec_reg')
   def __init__(self):
-    self.sgpr, self.vgpr = [0] * SGPR_COUNT, [[0] * VGPR_COUNT for _ in range(WAVE_SIZE)]
-    self.sgpr[EXEC_LO], self.scc, self.pc, self.literal, self._pend_sgpr = 0xffffffff, 0, 0, 0, {}
+    self.sgpr = [Reg(0) for _ in range(SGPR_COUNT)]
+    self.vgpr = [[Reg(0) for _ in range(VGPR_COUNT)] for _ in range(WAVE_SIZE)]
+    self.sgpr[EXEC_LO]._val = 0xffffffff
+    self.scc, self.pc, self.literal, self._pend_sgpr = 0, 0, 0, {}
+    # Reg wrappers for pseudocode access
+    self._scc_reg = Reg(0)
+    self._vcc_reg = self.sgpr[VCC_LO]
+    self._exec_reg = self.sgpr[EXEC_LO]
 
   @property
-  def vcc(self) -> int: return self.sgpr[VCC_LO] | (self.sgpr[VCC_HI] << 32)
+  def vcc(self) -> int: return self.sgpr[VCC_LO]._val | (self.sgpr[VCC_HI]._val << 32)
   @vcc.setter
-  def vcc(self, v: int): self.sgpr[VCC_LO], self.sgpr[VCC_HI] = v & MASK32, (v >> 32) & MASK32
+  def vcc(self, v: int): self.sgpr[VCC_LO]._val, self.sgpr[VCC_HI]._val = v & 0xffffffff, (v >> 32) & 0xffffffff
   @property
-  def exec_mask(self) -> int: return self.sgpr[EXEC_LO] | (self.sgpr[EXEC_HI] << 32)
+  def exec_mask(self) -> int: return self.sgpr[EXEC_LO]._val | (self.sgpr[EXEC_HI]._val << 32)
   @exec_mask.setter
-  def exec_mask(self, v: int): self.sgpr[EXEC_LO], self.sgpr[EXEC_HI] = v & MASK32, (v >> 32) & MASK32
+  def exec_mask(self, v: int): self.sgpr[EXEC_LO]._val, self.sgpr[EXEC_HI]._val = v & 0xffffffff, (v >> 32) & 0xffffffff
 
-  def rsgpr(self, i: int) -> int: return 0 if i == NULL else self.scc if i == SCC else self.sgpr[i] if i < SGPR_COUNT else 0
+  def rsgpr(self, i: int) -> int: return 0 if i == NULL else self.scc if i == SCC else self.sgpr[i]._val if i < SGPR_COUNT else 0
   def wsgpr(self, i: int, v: int):
-    if i < SGPR_COUNT and i != NULL: self.sgpr[i] = v & MASK32
+    if i < SGPR_COUNT and i != NULL: self.sgpr[i]._val = v & 0xffffffff
   def rsgpr64(self, i: int) -> int: return self.rsgpr(i) | (self.rsgpr(i+1) << 32)
-  def wsgpr64(self, i: int, v: int): self.wsgpr(i, v & MASK32); self.wsgpr(i+1, (v >> 32) & MASK32)
+  def wsgpr64(self, i: int, v: int): self.wsgpr(i, v & 0xffffffff); self.wsgpr(i+1, (v >> 32) & 0xffffffff)
 
-  def _rsrc_base(self, v: int, lane: int, consts):
-    if v < SGPR_COUNT: return self.sgpr[v]
+  def rsrc(self, v: int, lane: int) -> int:
+    if v < SGPR_COUNT: return self.sgpr[v]._val
     if v == SCC: return self.scc
-    if v < 255: return consts[v - 128]
+    if v < 255: return _INLINE_CONSTS[v - 128]
     if v == 255: return self.literal
-    return self.vgpr[lane][v - 256] if v <= 511 else 0
-  def rsrc(self, v: int, lane: int) -> int: return self._rsrc_base(v, lane, _INLINE_CONSTS)
-  def rsrc_f16(self, v: int, lane: int) -> int: return self._rsrc_base(v, lane, _INLINE_CONSTS_F16)
+    return self.vgpr[lane][v - 256]._val if v <= 511 else 0
+
+  def rsrc_reg(self, v: int, lane: int) -> Reg:
+    """Return the Reg object for a source operand."""
+    if v < SGPR_COUNT: return self.sgpr[v]
+    if v == SCC: self._scc_reg._val = self.scc; return self._scc_reg
+    if v < 255: return Reg(_INLINE_CONSTS[v - 128])
+    if v == 255: return Reg(self.literal)
+    return self.vgpr[lane][v - 256] if v <= 511 else Reg(0)
+
+  def rsrc_f16(self, v: int, lane: int) -> int:
+    """Read source operand for VOP3P packed f16 operations. Uses f16 inline constants."""
+    if v < SGPR_COUNT: return self.sgpr[v]._val
+    if v == SCC: return self.scc
+    if v < 255: return _INLINE_CONSTS_F16[v - 128]
+    if v == 255: return self.literal
+    return self.vgpr[lane][v - 256]._val if v <= 511 else 0
+
+  def rsrc_reg_f16(self, v: int, lane: int) -> Reg:
+    """Return Reg for VOP3P source. Inline constants are f16 in low 16 bits only."""
+    if v < SGPR_COUNT: return self.sgpr[v]
+    if v == SCC: self._scc_reg._val = self.scc; return self._scc_reg
+    if v < 255: return Reg(_INLINE_CONSTS_F16[v - 128])  # f16 inline constant
+    if v == 255: return Reg(self.literal)
+    return self.vgpr[lane][v - 256] if v <= 511 else Reg(0)
+
   def rsrc64(self, v: int, lane: int) -> int:
+    """Read 64-bit source operand. For inline constants, returns 64-bit representation."""
     if 128 <= v < 255: return _INLINE_CONSTS_F64[v - 128]
-    if v == 255: return self.literal  # literal is already shifted in from_bytes for 64-bit ops
+    if v == 255: return self.literal
     return self.rsrc(v, lane) | ((self.rsrc(v+1, lane) if v < VCC_LO or 256 <= v <= 511 else 0) << 32)
 
+  def rsrc_reg64(self, v: int, lane: int) -> Reg:
+    """Return Reg for 64-bit source operand. For inline constants, returns 64-bit f64 value."""
+    if 128 <= v < 255: return Reg(_INLINE_CONSTS_F64[v - 128])
+    if v == 255: return Reg(self.literal)
+    if v < SGPR_COUNT: return Reg(self.sgpr[v]._val | (self.sgpr[v+1]._val << 32))
+    if 256 <= v <= 511:
+      vgpr_idx = v - 256
+      return Reg(self.vgpr[lane][vgpr_idx]._val | (self.vgpr[lane][vgpr_idx + 1]._val << 32))
+    return Reg(0)
+
   def pend_sgpr_lane(self, reg: int, lane: int, val: int):
     if reg not in self._pend_sgpr: self._pend_sgpr[reg] = 0
     if val: self._pend_sgpr[reg] |= (1 << lane)
   def commit_pends(self):
-    for reg, val in self._pend_sgpr.items(): self.sgpr[reg] = val
+    for reg, val in self._pend_sgpr.items(): self.sgpr[reg]._val = val
     self._pend_sgpr.clear()
 
+# Instruction decode
+def decode_format(word: int) -> tuple[type[Inst] | None, bool]:
+  hi2 = (word >> 30) & 0x3
+  if hi2 == 0b11:
+    enc = (word >> 26) & 0xf
+    if enc == 0b1101: return SMEM, True
+    if enc == 0b0101:
+      op = (word >> 16) & 0x3ff
+      return (VOP3SD, True) if op in (288, 289, 290, 764, 765, 766, 767, 768, 769, 770) else (VOP3, True)
+    return {0b0011: (VOP3P, True), 0b0110: (DS, True), 0b0111: (FLAT, True), 0b0010: (VOPD, True)}.get(enc, (None, True))
+  if hi2 == 0b10:
+    enc = (word >> 23) & 0x7f
+    return {0b1111101: (SOP1, False), 0b1111110: (SOPC, False), 0b1111111: (SOPP, False)}.get(enc, (SOPK, False) if ((word >> 28) & 0xf) == 0b1011 else (SOP2, False))
+  enc = (word >> 25) & 0x7f
+  return (VOPC, False) if enc == 0b0111110 else (VOP1, False) if enc == 0b0111111 else (VOP2, False)
+
+def _unwrap(v) -> int: return v.val if isinstance(v, RawImm) else v.value if hasattr(v, 'value') else v
 
 def decode_program(data: bytes) -> Program:
   result: Program = {}
   i = 0
   while i < len(data):
-    try: inst_class = detect_format(data[i:])
-    except ValueError: break  # stop at invalid instruction (padding/metadata after code)
+    word = int.from_bytes(data[i:i+4], 'little')
+    inst_class, is_64 = decode_format(word)
     if inst_class is None: i += 4; continue
-    base_size = inst_class._size()
+    base_size = 8 if is_64 else 4
     # Pass enough data for potential 64-bit literal (base + 8 bytes max)
     inst = inst_class.from_bytes(data[i:i+base_size+8])
-    for name, val in inst._values.items():
-      if name != 'op': setattr(inst, name, unwrap(val))  # skip op to preserve property access
+    for name, val in inst._values.items(): setattr(inst, name, _unwrap(val))
+    # from_bytes already handles literal reading - only need fallback for cases it doesn't handle
+    if inst._literal is None:
+      has_literal = any(getattr(inst, fld, None) == 255 for fld in ('src0', 'src1', 'src2', 'ssrc0', 'ssrc1', 'srcx0', 'srcy0'))
+      if inst_class == VOP2 and inst.op in (44, 45, 55, 56): has_literal = True
+      if inst_class == VOPD and (inst.opx in (1, 2) or inst.opy in (1, 2)): has_literal = True
+      if inst_class == SOP2 and inst.op in (69, 70): has_literal = True
+      if has_literal:
+        # For 64-bit ops, the 32-bit literal is placed in HIGH 32 bits (low 32 bits = 0)
+        # Exception: some ops have mixed src sizes (e.g., V_LDEXP_F64 has 32-bit src1)
+        op_val = inst._values.get('op')
+        if hasattr(op_val, 'value'): op_val = op_val.value
+        is_64bit = inst_class is VOP3 and op_val in _VOP3_64BIT_OPS
+        # Don't treat literal as 64-bit if the op has 32-bit src1 and src1 is the literal
+        if is_64bit and op_val in _VOP3_64BIT_OPS_32BIT_SRC1 and getattr(inst, 'src1', None) == 255:
+          is_64bit = False
+        lit32 = int.from_bytes(data[i+base_size:i+base_size+4], 'little')
+        inst._literal = (lit32 << 32) if is_64bit else lit32
     inst._words = inst.size() // 4
     result[i // 4] = inst
     i += inst._words * 4
@@ -137,245 +228,298 @@ def decode_program(data: bytes) -> Program:
 def exec_scalar(st: WaveState, inst: Inst) -> int:
   """Execute scalar instruction. Returns PC delta or negative for special cases."""
   compiled = _get_compiled()
+  inst_type = type(inst)
 
-  # SOPP: special cases for control flow that has no pseudocode
-  if isinstance(inst, SOPP):
-    if inst.op == SOPPOp.S_ENDPGM: return -1
-    if inst.op == SOPPOp.S_BARRIER: return -2
+  # SOPP: control flow (not ALU)
+  if inst_type is SOPP:
+    op = inst.op
+    if op == SOPPOp.S_ENDPGM: return -1
+    if op == SOPPOp.S_BARRIER: return -2
+    if op == SOPPOp.S_BRANCH: return _sext(inst.simm16, 16)
+    if op == SOPPOp.S_CBRANCH_SCC0: return _sext(inst.simm16, 16) if st.scc == 0 else 0
+    if op == SOPPOp.S_CBRANCH_SCC1: return _sext(inst.simm16, 16) if st.scc == 1 else 0
+    if op == SOPPOp.S_CBRANCH_VCCZ: return _sext(inst.simm16, 16) if (st.vcc & 0xffffffff) == 0 else 0
+    if op == SOPPOp.S_CBRANCH_VCCNZ: return _sext(inst.simm16, 16) if (st.vcc & 0xffffffff) != 0 else 0
+    if op == SOPPOp.S_CBRANCH_EXECZ: return _sext(inst.simm16, 16) if st.exec_mask == 0 else 0
+    if op == SOPPOp.S_CBRANCH_EXECNZ: return _sext(inst.simm16, 16) if st.exec_mask != 0 else 0
+    # Valid SOPP range is 0-61 (max defined opcode); anything above is invalid
+    if op > 61: raise NotImplementedError(f"Invalid SOPP opcode {op}")
+    return 0  # waits, hints, nops
 
   # SMEM: memory loads (not ALU)
-  if isinstance(inst, SMEM):
+  if inst_type is SMEM:
     addr = st.rsgpr64(inst.sbase * 2) + _sext(inst.offset, 21)
     if inst.soffset not in (NULL, 0x7f): addr += st.rsrc(inst.soffset, 0)
     if (cnt := SMEM_LOAD.get(inst.op)) is None: raise NotImplementedError(f"SMEM op {inst.op}")
-    for i in range(cnt): st.wsgpr(inst.sdata + i, mem_read((addr + i * 4) & MASK64, 4))
+    for i in range(cnt): st.wsgpr(inst.sdata + i, mem_read((addr + i * 4) & 0xffffffffffffffff, 4))
     return 0
 
+  # SOP1: special handling for ops not in pseudocode
+  if inst_type is SOP1:
+    op = SOP1Op(inst.op)
+    # S_GETPC_B64: Get program counter (PC is stored as byte offset, convert from words)
+    if op == SOP1Op.S_GETPC_B64:
+      pc_bytes = st.pc * 4  # PC is in words, convert to bytes
+      st.wsgpr64(inst.sdst, pc_bytes)
+      return 0
+    # S_SETPC_B64: Set program counter to source value (indirect jump)
+    # Returns delta such that st.pc + inst_words + delta = target_words
+    if op == SOP1Op.S_SETPC_B64:
+      target_bytes = st.rsrc64(inst.ssrc0, 0)
+      target_words = target_bytes // 4
+      inst_words = 1  # SOP1 is always 1 word
+      return target_words - st.pc - inst_words
+
   # Get op enum and lookup compiled function
-  if isinstance(inst, SOP1): ssrc0, sdst = inst.ssrc0, inst.sdst
-  elif isinstance(inst, SOP2): ssrc0, sdst = inst.ssrc0, inst.sdst
-  elif isinstance(inst, SOPC): ssrc0, sdst = inst.ssrc0, None
-  elif isinstance(inst, SOPK): ssrc0, sdst = inst.sdst, inst.sdst  # sdst is both src and dst
-  elif isinstance(inst, SOPP): ssrc0, sdst = None, None
-  else: raise NotImplementedError(f"Unknown scalar type {type(inst)}")
+  if inst_type is SOP1: op_cls, ssrc0, sdst = SOP1Op, inst.ssrc0, inst.sdst
+  elif inst_type is SOP2: op_cls, ssrc0, sdst = SOP2Op, inst.ssrc0, inst.sdst
+  elif inst_type is SOPC: op_cls, ssrc0, sdst = SOPCOp, inst.ssrc0, None
+  elif inst_type is SOPK: op_cls, ssrc0, sdst = SOPKOp, inst.sdst, inst.sdst  # sdst is both src and dst
+  else: raise NotImplementedError(f"Unknown scalar type {inst_type}")
 
-  # SOPP has gaps in the opcode enum - treat unknown opcodes as no-ops
-  try: op = inst.op
-  except ValueError:
-    if isinstance(inst, SOPP): return 0
-    raise
-  fn = compiled.get(type(op), {}).get(op)
-  if fn is None:
-    # SOPP instructions without pseudocode (waits, hints, nops) are no-ops
-    if isinstance(inst, SOPP): return 0
-    raise NotImplementedError(f"{op.name} not in pseudocode")
+  op = op_cls(inst.op)
+  fn = compiled.get(op_cls, {}).get(op)
+  if fn is None: raise NotImplementedError(f"{op.name} not in pseudocode")
 
-  # Build context - use inst methods to determine operand sizes
-  s0 = st.rsrc64(ssrc0, 0) if inst.is_src_64(0) else (st.rsrc(ssrc0, 0) if not isinstance(inst, (SOPK, SOPP)) else (st.rsgpr(inst.sdst) if isinstance(inst, SOPK) else 0))
-  s1 = st.rsrc64(inst.ssrc1, 0) if inst.is_src_64(1) else (st.rsrc(inst.ssrc1, 0) if isinstance(inst, (SOP2, SOPC)) else inst.simm16 if isinstance(inst, SOPK) else 0)
-  d0 = st.rsgpr64(sdst) if inst.dst_regs() == 2 and sdst is not None else (st.rsgpr(sdst) if sdst is not None else 0)
-  literal = inst.simm16 if isinstance(inst, (SOPK, SOPP)) else st.literal
+  # Build context - handle 64-bit ops that need 64-bit source reads
+  # 64-bit source ops: name ends with _B64, _I64, _U64 or contains _U64, _I64 before last underscore
+  is_64bit_s0 = op.name.endswith(('_B64', '_I64', '_U64')) or '_U64_' in op.name or '_I64_' in op.name
+  is_64bit_s0s1 = op_cls is SOPCOp and op in (SOPCOp.S_CMP_EQ_U64, SOPCOp.S_CMP_LG_U64)
+  s0 = st.rsrc64(ssrc0, 0) if is_64bit_s0 or is_64bit_s0s1 else (st.rsrc(ssrc0, 0) if inst_type != SOPK else st.rsgpr(inst.sdst))
+  is_64bit_sop2 = is_64bit_s0 and inst_type is SOP2
+  s1 = st.rsrc64(inst.ssrc1, 0) if (is_64bit_sop2 or is_64bit_s0s1) else (st.rsrc(inst.ssrc1, 0) if inst_type in (SOP2, SOPC) else inst.simm16 if inst_type is SOPK else 0)
+  d0 = st.rsgpr64(sdst) if (is_64bit_s0 or is_64bit_s0s1) and sdst is not None else (st.rsgpr(sdst) if sdst is not None else 0)
+  literal = inst.simm16 if inst_type is SOPK else st.literal
 
-  # Create Reg objects for compiled function - mask VCC/EXEC to 32 bits for wave32
-  result = fn(Reg(s0), Reg(s1), None, Reg(d0), Reg(st.scc), Reg(st.vcc & MASK32), 0, Reg(st.exec_mask & MASK32), literal, None, PC=Reg(st.pc * 4))
+  # Create Reg objects for new calling convention
+  S0, S1, S2, D0 = Reg(s0), Reg(s1), Reg(0), Reg(d0)
+  SCC, VCC, EXEC = Reg(st.scc), Reg(st.vcc), Reg(st.exec_mask)
 
-  # Apply results - extract values from returned Reg objects
-  if sdst is not None and 'D0' in result:
-    (st.wsgpr64 if inst.dst_regs() == 2 else st.wsgpr)(sdst, result['D0']._val)
-  if 'SCC' in result: st.scc = result['SCC']._val & 1
-  if 'EXEC' in result: st.exec_mask = result['EXEC']._val
-  if 'PC' in result:
-    # Convert absolute byte address to word delta
-    pc_val = result['PC']._val
-    new_pc = pc_val if pc_val < 0x8000000000000000 else pc_val - 0x10000000000000000
-    new_pc_words = new_pc // 4
-    return new_pc_words - st.pc - 1  # -1 because emulator adds inst_words (1 for scalar)
+  # Execute compiled function - fn(S0, S1, S2, D0, SCC, VCC, laneId, EXEC, SIMM16, VGPR, SRC0, VDST)
+  fn(S0, S1, S2, D0, SCC, VCC, 0, EXEC, Reg(literal), None, 0, 0)
+
+  # Apply results from Reg objects
+  is_64bit_d0 = is_64bit_s0 or is_64bit_s0s1
+  if sdst is not None:
+    if is_64bit_d0:
+      st.wsgpr64(sdst, D0._val)
+    else:
+      st.wsgpr(sdst, D0._val)
+  st.scc = SCC._val
+  st.exec_mask = EXEC._val
   return 0
 
-def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = None) -> None:
-  """Execute vector instruction for one lane."""
+def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = None,
+                d0_override: 'Reg | None' = None, vcc_override: 'Reg | None' = None) -> None:
+  """Execute vector instruction for one lane.
+  d0_override: For VOPC/VOP3-VOPC, use this Reg instead of st.sgpr[vdst] for D0 output.
+  vcc_override: For VOP3SD, use this Reg instead of st.sgpr[sdst] for VCC output.
+  """
   compiled = _get_compiled()
-  V = st.vgpr[lane]
+  inst_type, V = type(inst), st.vgpr[lane]
 
   # Memory ops (not ALU pseudocode)
-  if isinstance(inst, FLAT):
+  if inst_type is FLAT:
     op, addr_reg, data_reg, vdst, offset, saddr = inst.op, inst.addr, inst.data, inst.vdst, _sext(inst.offset, 13), inst.saddr
-    addr = V[addr_reg] | (V[addr_reg+1] << 32)
-    addr = (st.rsgpr64(saddr) + V[addr_reg] + offset) & MASK64 if saddr not in (NULL, 0x7f) else (addr + offset) & MASK64
+    addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+    addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
     if op in FLAT_LOAD:
       cnt, sz, sign = FLAT_LOAD[op]
-      for i in range(cnt): val = mem_read(addr + i * sz, sz); V[vdst + i] = _sext(val, sz * 8) & MASK32 if sign else val
+      for i in range(cnt): val = mem_read(addr + i * sz, sz); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
     elif op in FLAT_STORE:
       cnt, sz = FLAT_STORE[op]
-      for i in range(cnt): mem_write(addr + i * sz, sz, V[data_reg + i] & ((1 << (sz * 8)) - 1))
+      for i in range(cnt): mem_write(addr + i * sz, sz, V[data_reg + i]._val & ((1 << (sz * 8)) - 1))
     elif op in FLAT_D16_LOAD:
       sz, sign, hi = FLAT_D16_LOAD[op]
       val = mem_read(addr, sz)
       if sign: val = _sext(val, sz * 8) & 0xffff
-      V[vdst] = _dst16(V[vdst], val, hi)
+      if hi: V[vdst]._val = (V[vdst]._val & 0xffff) | (val << 16)
+      else: V[vdst]._val = (V[vdst]._val & 0xffff0000) | (val & 0xffff)
     elif op in FLAT_D16_STORE:
       sz, hi = FLAT_D16_STORE[op]
-      mem_write(addr, sz, _src16(V[data_reg], hi) & ((1 << (sz * 8)) - 1))
+      val = (V[data_reg]._val >> 16) & 0xffff if hi else V[data_reg]._val & 0xffff
+      mem_write(addr, sz, val & ((1 << (sz * 8)) - 1))
     else: raise NotImplementedError(f"FLAT op {op}")
     return
 
-  if isinstance(inst, DS):
-    fn = compiled.get(DSOp, {}).get(inst.op)
-    if fn is None: raise NotImplementedError(f"DS op {inst.op.name} not in pseudocode")
-    # Prepare data registers as lists of dwords
-    data0 = [V[inst.data0 + i] for i in range(4)]  # up to 4 dwords
-    data1 = [V[inst.data1 + i] for i in range(4)] if inst.data1 else [0, 0, 0, 0]
-    result = fn(lds, V[inst.addr], data0, data1, inst.vdst, inst.offset0, inst.offset1)
-    # Write results for loads
-    if 'vdst' in result:
-      for i, val in enumerate(result['vdst']): V[inst.vdst + i] = val & MASK32
+  if inst_type is DS:
+    op, addr, vdst = inst.op, (V[inst.addr]._val + inst.offset0) & 0xffff, inst.vdst
+    if op in DS_LOAD:
+      cnt, sz, sign = DS_LOAD[op]
+      for i in range(cnt): val = int.from_bytes(lds[addr+i*sz:addr+i*sz+sz], 'little'); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in DS_STORE:
+      cnt, sz = DS_STORE[op]
+      for i in range(cnt): lds[addr+i*sz:addr+i*sz+sz] = (V[inst.data0 + i]._val & ((1 << (sz * 8)) - 1)).to_bytes(sz, 'little')
+    else: raise NotImplementedError(f"DS op {op}")
     return
 
-  # VOPD: dual-issue, execute two ops simultaneously (read all inputs before writes)
-  if isinstance(inst, VOPD):
+  # VOPD: dual-issue, execute two ops using VOP2/VOP3 compiled functions
+  if inst_type is VOPD:
     vdsty = (inst.vdsty << 1) | ((inst.vdstx & 1) ^ 1)
-    inputs = [(inst.opx, st.rsrc(inst.srcx0, lane), V[inst.vsrcx1], V[inst.vdstx], inst.vdstx),
-              (inst.opy, st.rsrc(inst.srcy0, lane), V[inst.vsrcy1], V[vdsty], vdsty)]
-    def exec_vopd(vopd_op, s0, s1, d0):
-      op = _VOPD_TO_VOP[vopd_op]
-      return compiled[type(op)][op](Reg(s0), Reg(s1), None, Reg(d0), Reg(st.scc), Reg(st.vcc), lane, Reg(st.exec_mask), st.literal, None)['D0']._val
-    for vopd_op, s0, s1, d0, dst in inputs: V[dst] = exec_vopd(vopd_op, s0, s1, d0)
+    # Read all source operands BEFORE any writes (dual-issue semantics)
+    sx0, sx1 = Reg(st.rsrc(inst.srcx0, lane)), Reg(V[inst.vsrcx1]._val)
+    sy0, sy1 = Reg(st.rsrc(inst.srcy0, lane)), Reg(V[inst.vsrcy1]._val)
+    dx0, dy0 = Reg(V[inst.vdstx]._val), Reg(V[vdsty]._val)
+    st._scc_reg._val = st.scc
+    if (op_x := _VOPD_TO_VOP.get(inst.opx)):
+      if (fn_x := compiled.get(type(op_x), {}).get(op_x)):
+        fn_x(sx0, sx1, Reg(0), dx0, st._scc_reg, st.sgpr[VCC_LO], lane, st.sgpr[EXEC_LO], Reg(st.literal), None, Reg(0), Reg(inst.vdstx))
+    if (op_y := _VOPD_TO_VOP.get(inst.opy)):
+      if (fn_y := compiled.get(type(op_y), {}).get(op_y)):
+        fn_y(sy0, sy1, Reg(0), dy0, st._scc_reg, st.sgpr[VCC_LO], lane, st.sgpr[EXEC_LO], Reg(st.literal), None, Reg(0), Reg(vdsty))
+    V[inst.vdstx]._val, V[vdsty]._val = dx0._val, dy0._val
+    st.scc = st._scc_reg._val
     return
 
-  # VOP3SD: has extra scalar dest for carry output
-  if isinstance(inst, VOP3SD):
-    fn = compiled[VOP3SDOp][inst.op]
-    # Read sources based on register counts from inst properties
-    def rsrc_n(src, regs): return st.rsrc64(src, lane) if regs == 2 else st.rsrc(src, lane)
-    s0, s1, s2 = rsrc_n(inst.src0, inst.src_regs(0)), rsrc_n(inst.src1, inst.src_regs(1)), rsrc_n(inst.src2, inst.src_regs(2))
-    # Carry-in ops use src2 as carry bitmask instead of VCC
-    vcc = st.rsgpr64(inst.src2) if 'CO_CI' in inst.op_name else st.vcc
-    result = fn(Reg(s0), Reg(s1), Reg(s2), Reg(V[inst.vdst]), Reg(st.scc), Reg(vcc), lane, Reg(st.exec_mask), st.literal, None)
-    d0_val = result['D0']._val
-    V[inst.vdst] = d0_val & MASK32
-    if inst.dst_regs() == 2: V[inst.vdst + 1] = (d0_val >> 32) & MASK32
-    if 'VCC' in result: st.pend_sgpr_lane(inst.sdst, lane, (result['VCC']._val >> lane) & 1)
-    return
-
-  # Get op enum and sources (None means "no source" for that operand)
-  # dst_hi: for VOP1/VOP2 16-bit dst ops, bit 7 of vdst indicates .h (high 16-bit) destination
-  dst_hi = False
-  if isinstance(inst, VOP1):
+  # Determine instruction format and get function
+  is_vop3_vopc = False
+  is_readlane = False
+  if inst_type is VOP1:
     if inst.op == VOP1Op.V_NOP: return
-    src0, src1, src2 = inst.src0, None, None
-    dst_hi = (inst.vdst & 0x80) != 0 and inst.is_dst_16()
-    vdst = inst.vdst & 0x7f if inst.is_dst_16() else inst.vdst
-  elif isinstance(inst, VOP2):
-    src0, src1, src2 = inst.src0, inst.vsrc1 + 256, None
-    dst_hi = (inst.vdst & 0x80) != 0 and inst.is_dst_16()
-    vdst = inst.vdst & 0x7f if inst.is_dst_16() else inst.vdst
-  elif isinstance(inst, VOP3):
-    # VOP3 ops 0-255 are VOPC comparisons encoded as VOP3 - inst.op returns VOPCOp for these
-    src0, src1, src2, vdst = inst.src0, inst.src1, (None if inst.op.value < 256 else inst.src2), inst.vdst
-  elif isinstance(inst, VOPC):
-    # For 16-bit VOPC, vsrc1 uses same encoding as VOP2 16-bit: bit 7 selects hi(1) or lo(0) half
-    # vsrc1 field is 8 bits: [6:0] = VGPR index, [7] = hi flag
-    src0, src1, src2, vdst = inst.src0, inst.vsrc1 + 256, None, VCC_LO
-  elif isinstance(inst, VOP3P):
-    # VOP3P: Packed 16-bit operations using compiled functions
-    # WMMA: wave-level matrix multiply-accumulate (special handling - needs cross-lane access)
-    if 'WMMA' in inst.op_name:
-      if lane == 0:  # Only execute once per wave, write results for all lanes
-        exec_wmma(st, inst, inst.op)
+    op_cls, op, src0, src1, src2, vdst = VOP1Op, VOP1Op(inst.op), inst.src0, None, None, inst.vdst
+    # V_READFIRSTLANE_B32 writes to SGPR, not VGPR
+    is_readlane = inst.op == VOP1Op.V_READFIRSTLANE_B32
+  elif inst_type is VOP2:
+    op_cls, op, src0, src1, src2, vdst = VOP2Op, VOP2Op(inst.op), inst.src0, inst.vsrc1 + 256, None, inst.vdst
+  elif inst_type is VOP3:
+    if inst.op < 256:
+      # VOP3-encoded VOPC - destination is an SGPR (vdst field)
+      op_cls, op, src0, src1, src2, vdst = VOPCOp, VOPCOp(inst.op), inst.src0, inst.src1, None, inst.vdst
+      is_vop3_vopc = True
+    else:
+      op_cls, op, src0, src1, src2, vdst = VOP3Op, VOP3Op(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+      # V_READFIRSTLANE_B32 and V_READLANE_B32 write to SGPR
+      is_readlane = inst.op in (VOP3Op.V_READFIRSTLANE_B32, VOP3Op.V_READLANE_B32)
+  elif inst_type is VOP3SD:
+    op_cls, op, src0, src1, src2, vdst = VOP3SDOp, VOP3SDOp(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+  elif inst_type is VOPC:
+    op_cls, op, src0, src1, src2, vdst = VOPCOp, VOPCOp(inst.op), inst.src0, inst.vsrc1 + 256, None, VCC_LO
+  elif inst_type is VOP3P:
+    op_cls, op, src0, src1, src2, vdst = VOP3POp, VOP3POp(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+    # WMMA instructions are handled specially (only execute for lane 0)
+    if op in (VOP3POp.V_WMMA_F32_16X16X16_F16, VOP3POp.V_WMMA_F16_16X16X16_F16):
+      if lane == 0: exec_wmma(st, inst, op)
       return
-    # V_FMA_MIX: Mixed precision FMA - opsel_hi controls f32(0) vs f16(1), opsel selects which f16 half
-    # Handle inline because abs/neg must be applied AFTER type conversion
-    if inst.op in (VOP3POp.V_FMA_MIX_F32, VOP3POp.V_FMA_MIXLO_F16, VOP3POp.V_FMA_MIXHI_F16):
-      opsel, opsel_hi, opsel_hi2 = getattr(inst, 'opsel', 0), getattr(inst, 'opsel_hi', 0), getattr(inst, 'opsel_hi2', 0)
-      neg, abs_ = getattr(inst, 'neg', 0), getattr(inst, 'neg_hi', 0)  # neg_hi reused as abs for FMA_MIX
-      raws = [st.rsrc(inst.src0, lane), st.rsrc(inst.src1, lane), st.rsrc(inst.src2, lane) if inst.src2 is not None else 0]
-      is_f16 = [opsel_hi & 1, opsel_hi & 2, opsel_hi2]
-      srcs = [_f16(_src16(raws[i], bool(opsel & (1<<i)))) if is_f16[i] else _f32(raws[i]) for i in range(3)]
-      for i in range(3):
-        if abs_ & (1<<i): srcs[i] = abs(srcs[i])
-        if neg & (1<<i): srcs[i] = -srcs[i]
-      result_f = srcs[0] * srcs[1] + srcs[2]
-      V = st.vgpr[lane]
-      V[inst.vdst] = _i32(result_f) if inst.op == VOP3POp.V_FMA_MIX_F32 else _dst16(V[inst.vdst], _i16(result_f), inst.op == VOP3POp.V_FMA_MIXHI_F16)
-      return
-    # VOP3P packed ops: opsel selects halves for lo, opsel_hi for hi; neg toggles f16 sign
-    raws = [st.rsrc_f16(inst.src0, lane), st.rsrc_f16(inst.src1, lane), st.rsrc_f16(inst.src2, lane) if inst.src2 is not None else 0]
-    opsel, opsel_hi, opsel_hi2 = getattr(inst, 'opsel', 0), getattr(inst, 'opsel_hi', 3), getattr(inst, 'opsel_hi2', 1)
-    neg, neg_hi = getattr(inst, 'neg', 0), getattr(inst, 'neg_hi', 0)
-    hi_sels = [opsel_hi & 1, opsel_hi & 2, opsel_hi2]
-    srcs = [((_src16(raws[i], hi_sels[i]) ^ (0x8000 if neg_hi & (1<<i) else 0)) << 16) |
-            (_src16(raws[i], opsel & (1<<i)) ^ (0x8000 if neg & (1<<i) else 0)) for i in range(3)]
-    result = compiled[VOP3POp][inst.op](Reg(srcs[0]), Reg(srcs[1]), Reg(srcs[2]), Reg(0), Reg(st.scc), Reg(st.vcc), lane, Reg(st.exec_mask), st.literal, None)
-    st.vgpr[lane][inst.vdst] = result['D0']._val & MASK32
-    return
-  else: raise NotImplementedError(f"Unknown vector type {type(inst)}")
+  else: raise NotImplementedError(f"Unknown vector type {inst_type}")
 
-  op_cls = type(inst.op)
-  if (fn := compiled.get(op_cls, {}).get(inst.op)) is None: raise NotImplementedError(f"{inst.op_name} not in pseudocode")
+  fn = compiled.get(op_cls, {}).get(op)
+  if fn is None: raise NotImplementedError(f"{op.name} not in pseudocode")
 
-  # Read sources (with VOP3 modifiers if applicable)
-  neg, abs_ = (getattr(inst, 'neg', 0), getattr(inst, 'abs', 0)) if isinstance(inst, VOP3) else (0, 0)
-  opsel = getattr(inst, 'opsel', 0) if isinstance(inst, VOP3) else 0
-  def mod_src(val: int, idx: int, is64=False) -> int:
-    to_f, to_i = (_f64, _i64) if is64 else (_f32, _i32)
-    if (abs_ >> idx) & 1: val = to_i(abs(to_f(val)))
-    if (neg >> idx) & 1: val = to_i(-to_f(val))
-    return val
+  # Build source Regs - get the actual register or create temp for inline constants
+  # VOP3P uses f16 inline constants (16-bit value in low half only)
+  if inst_type is VOP3P:
+    S0 = st.rsrc_reg_f16(src0, lane)
+    S1 = st.rsrc_reg_f16(src1, lane) if src1 is not None else Reg(0)
+    S2 = st.rsrc_reg_f16(src2, lane) if src2 is not None else Reg(0)
+    # Apply op_sel_hi modifiers: control which half is used for hi-half computation
+    # opsel_hi[0]=0 means src0 hi comes from lo half, =1 means from hi half (default)
+    # opsel_hi[1]=0 means src1 hi comes from lo half, =1 means from hi half (default)
+    # opsel_hi2=0 means src2 hi comes from lo half, =1 means from hi half (default)
+    opsel_hi = getattr(inst, 'opsel_hi', 3)  # default 0b11
+    opsel_hi2 = getattr(inst, 'opsel_hi2', 1)  # default 1
+    # If opsel_hi bit is 0, replicate lo half to hi half
+    if not (opsel_hi & 1):  # src0 hi from lo
+      lo = S0._val & 0xffff
+      S0 = Reg((lo << 16) | lo)
+    if not (opsel_hi & 2):  # src1 hi from lo
+      lo = S1._val & 0xffff
+      S1 = Reg((lo << 16) | lo)
+    if not opsel_hi2:  # src2 hi from lo
+      lo = S2._val & 0xffff
+      S2 = Reg((lo << 16) | lo)
+  else:
+    # Check if this is a 64-bit F64 op - needs 64-bit source reads for f64 operands
+    # V_LDEXP_F64: S0 is f64, S1 is i32 (exponent)
+    # V_ADD_F64, V_MUL_F64, etc: S0 and S1 are f64
+    # VOP1 F64 ops (V_TRUNC_F64, V_FLOOR_F64, etc): S0 is f64
+    is_f64_op = hasattr(op, 'name') and '_F64' in op.name
+    is_ldexp_f64 = hasattr(op, 'name') and op.name == 'V_LDEXP_F64'
+    if is_f64_op:
+      S0 = st.rsrc_reg64(src0, lane)
+      # V_LDEXP_F64: S1 is i32 exponent, not f64
+      if is_ldexp_f64:
+        S1 = st.rsrc_reg(src1, lane) if src1 is not None else Reg(0)
+      else:
+        S1 = st.rsrc_reg64(src1, lane) if src1 is not None else Reg(0)
+      S2 = st.rsrc_reg64(src2, lane) if src2 is not None else Reg(0)
+    else:
+      S0 = st.rsrc_reg(src0, lane)
+      S1 = st.rsrc_reg(src1, lane) if src1 is not None else Reg(0)
+      S2 = st.rsrc_reg(src2, lane) if src2 is not None else Reg(0)
+    # VOP3SD V_MAD_U64_U32 and V_MAD_I64_I32 need S2 as 64-bit from VGPR pair
+    if inst_type is VOP3SD and op in (VOP3SDOp.V_MAD_U64_U32, VOP3SDOp.V_MAD_I64_I32) and src2 is not None:
+      if 256 <= src2 <= 511:  # VGPR
+        vgpr_idx = src2 - 256
+        S2 = Reg(V[vgpr_idx]._val | (V[vgpr_idx + 1]._val << 32))
 
-  # Use inst methods to determine operand sizes (inst.is_src_16, inst.is_src_64, etc.)
-  is_vop2_16bit = isinstance(inst, VOP2) and inst.is_16bit()
+  # Apply source modifiers (neg, abs) for VOP3/VOP3SD
+  if inst_type in (VOP3, VOP3SD):
+    neg, abs_mod = getattr(inst, 'neg', 0), getattr(inst, 'abs', 0)
+    if neg or abs_mod:
+      # Apply to f32 values - need to handle as float
+      import struct
+      def apply_mods(reg, neg_bit, abs_bit):
+        val = reg._val
+        f = struct.unpack('<f', struct.pack('<I', val & 0xffffffff))[0]
+        if abs_bit: f = abs(f)
+        if neg_bit: f = -f
+        return Reg(struct.unpack('<I', struct.pack('<f', f))[0])
+      if neg & 1 or abs_mod & 1: S0 = apply_mods(S0, neg & 1, abs_mod & 1)
+      if neg & 2 or abs_mod & 2: S1 = apply_mods(S1, neg & 2, abs_mod & 2)
+      if neg & 4 or abs_mod & 4: S2 = apply_mods(S2, neg & 4, abs_mod & 4)
 
-  # Read sources based on register counts and dtypes from inst properties
-  def read_src(src, idx, regs, is_src_16):
-    if src is None: return 0
-    if regs == 2: return mod_src(st.rsrc64(src, lane), idx, is64=True)
-    if is_src_16 and isinstance(inst, VOP3):
-      raw = st.rsrc_f16(src, lane) if 128 <= src < 255 else st.rsrc(src, lane)
-      val = _src16(raw, bool(opsel & (1 << idx)))
-      if abs_ & (1 << idx): val &= 0x7fff
-      if neg & (1 << idx): val ^= 0x8000
-      return val
-    if is_src_16 and isinstance(inst, (VOP1, VOP2, VOPC)):
-      if src >= 256: return _src16(mod_src(st.rsrc(_vgpr_masked(src), lane), idx), _vgpr_hi(src))
-      return mod_src(st.rsrc_f16(src, lane), idx) & 0xffff
-    return mod_src(st.rsrc(src, lane), idx)
+  # Apply opsel for VOP3 f16 operations - select which half to use
+  # opsel[0]: src0, opsel[1]: src1, opsel[2]: src2 (0=lo, 1=hi)
+  if inst_type is VOP3:
+    opsel = getattr(inst, 'opsel', 0)
+    if opsel:
+      # If opsel bit is set, swap lo and hi so that .f16 reads the hi half
+      if opsel & 1:  # src0 from hi
+        S0 = Reg(((S0._val >> 16) & 0xffff) | (S0._val << 16))
+      if opsel & 2:  # src1 from hi
+        S1 = Reg(((S1._val >> 16) & 0xffff) | (S1._val << 16))
+      if opsel & 4:  # src2 from hi
+        S2 = Reg(((S2._val >> 16) & 0xffff) | (S2._val << 16))
 
-  s0 = read_src(src0, 0, inst.src_regs(0), inst.is_src_16(0))
-  s1 = read_src(src1, 1, inst.src_regs(1), inst.is_src_16(1)) if src1 is not None else 0
-  s2 = read_src(src2, 2, inst.src_regs(2), inst.is_src_16(2)) if src2 is not None else 0
-  # Read destination (accumulator for VOP2 f16, 64-bit for 64-bit ops)
-  d0 = _src16(V[vdst], dst_hi) if is_vop2_16bit else (V[vdst] | (V[vdst + 1] << 32)) if inst.dst_regs() == 2 else V[vdst]
+  # For VOPC and VOP3-encoded VOPC, D0 is an SGPR (VCC_LO for VOPC, vdst for VOP3 VOPC)
+  # V_READFIRSTLANE_B32 and V_READLANE_B32 also write to SGPR
+  # Use d0_override if provided (for batch execution with shared output register)
+  is_vopc = inst_type is VOPC or (inst_type is VOP3 and is_vop3_vopc)
+  if is_vopc:
+    D0 = d0_override if d0_override is not None else st.sgpr[VCC_LO if inst_type is VOPC else vdst]
+  elif is_readlane:
+    D0 = st.sgpr[vdst]
+  else:
+    D0 = V[vdst]
 
-  # V_CNDMASK_B32/B16: VOP3 encoding uses src2 as mask (not VCC); VOP2 uses VCC implicitly
-  # Pass the correct mask as vcc to the function so pseudocode VCC.u64[laneId] works correctly
-  vcc_for_fn = st.rsgpr64(src2) if inst.op in (VOP3Op.V_CNDMASK_B32, VOP3Op.V_CNDMASK_B16) and isinstance(inst, VOP3) and src2 is not None and src2 < 256 else st.vcc
+  # Execute compiled function - D0 is modified in place
+  st._scc_reg._val = st.scc
+  # For VOP3SD, pass sdst register as VCC parameter (carry-out destination)
+  # Use vcc_override if provided (for batch execution with shared output register)
+  # For VOP3 V_CNDMASK_B32, src2 specifies the condition selector (not VCC)
+  if inst_type is VOP3SD:
+    vcc_reg = vcc_override if vcc_override is not None else st.sgpr[inst.sdst]
+  elif inst_type is VOP3 and op == VOP3Op.V_CNDMASK_B32 and src2 is not None:
+    vcc_reg = st.rsrc_reg(src2, lane)  # Use src2 as condition
+  else:
+    vcc_reg = st.sgpr[VCC_LO]
+  # SRC0/VDST are VGPR indices (0-255), not hardware encoding (256-511)
+  src0_idx = (src0 - 256) if src0 and src0 >= 256 else (src0 if src0 else 0)
+  result = fn(S0, S1, S2, D0, st._scc_reg, vcc_reg, lane, st.sgpr[EXEC_LO], Reg(st.literal), st.vgpr, Reg(src0_idx), Reg(vdst))
+  st.scc = st._scc_reg._val
 
-  # Execute compiled function - pass src0_idx and vdst_idx for lane instructions
-  # For VGPR access: src0 index is the VGPR number (src0 - 256 if VGPR, else src0 for SGPR)
-  src0_idx = (src0 - 256) if src0 is not None and src0 >= 256 else (src0 if src0 is not None else 0)
-  result = fn(Reg(s0), Reg(s1), Reg(s2), Reg(d0), Reg(st.scc), Reg(vcc_for_fn), lane, Reg(st.exec_mask), st.literal, st.vgpr, src0_idx, vdst)
+  # Handle special results
+  if result:
+    if 'vgpr_write' in result:
+      wr_lane, wr_idx, wr_val = result['vgpr_write']
+      st.vgpr[wr_lane][wr_idx]._val = wr_val
 
-  # Apply results - extract values from returned Reg objects
-  if 'vgpr_write' in result:
-    # Lane instruction wrote to VGPR: (lane, vgpr_idx, value)
-    wr_lane, wr_idx, wr_val = result['vgpr_write']
-    st.vgpr[wr_lane][wr_idx] = wr_val
-  if 'VCC' in result:
-    # VOP2 carry ops write to VCC implicitly; VOPC/VOP3 write to vdst
-    st.pend_sgpr_lane(VCC_LO if isinstance(inst, VOP2) and 'CO_CI' in inst.op_name else vdst, lane, (result['VCC']._val >> lane) & 1)
-  if 'EXEC' in result:
-    # V_CMPX instructions write to EXEC per-lane (not to vdst)
-    st.pend_sgpr_lane(EXEC_LO, lane, (result['EXEC']._val >> lane) & 1)
-  elif op_cls is VOPCOp:
-    # VOPC comparison result stored in D0 bitmask, extract lane bit (non-CMPX only)
-    st.pend_sgpr_lane(vdst, lane, (result['D0']._val >> lane) & 1)
-  if op_cls is not VOPCOp and 'vgpr_write' not in result:
-    writes_to_sgpr = 'READFIRSTLANE' in inst.op_name or 'READLANE' in inst.op_name
-    d0_val = result['D0']._val
-    if writes_to_sgpr: st.wsgpr(vdst, d0_val & MASK32)
-    elif inst.dst_regs() == 2: V[vdst], V[vdst + 1] = d0_val & MASK32, (d0_val >> 32) & MASK32
-    elif inst.is_dst_16(): V[vdst] = _dst16(V[vdst], d0_val, bool(opsel & 8) if isinstance(inst, VOP3) else dst_hi)
-    else: V[vdst] = d0_val & MASK32
+  # 64-bit destination: write high 32 bits to next VGPR (determined from op name)
+  is_64bit_dst = not is_vopc and not is_readlane and hasattr(op, 'name') and \
+                 any(s in op.name for s in ('_B64', '_I64', '_U64', '_F64'))
+  if is_64bit_dst:
+    V[vdst + 1]._val = (D0._val >> 32) & 0xffffffff
+    D0._val = D0._val & 0xffffffff  # Keep only low 32 bits in D0
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # WMMA (Wave Matrix Multiply-Accumulate)
@@ -384,41 +528,187 @@ def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = No
 def exec_wmma(st: WaveState, inst, op: VOP3POp) -> None:
   """Execute WMMA instruction - 16x16x16 matrix multiply across the wave."""
   src0, src1, src2, vdst = inst.src0, inst.src1, inst.src2, inst.vdst
-  # Read 16x16 f16 matrix from 16 lanes × 8 VGPRs (2 f16 per VGPR)
-  def read_f16_mat(src):
-    return [f for l in range(16) for r in range(8) for v in [st.vgpr[l][src-256+r] if src >= 256 else st.rsgpr(src+r)] for f in [_f16(v&0xffff), _f16((v>>16)&0xffff)]]
-  mat_a, mat_b = read_f16_mat(src0), read_f16_mat(src1)
+  # Read matrix A (16x16 f16/bf16) from lanes 0-15, VGPRs src0 to src0+7 (2 f16 per VGPR = 16 values per lane)
+  # Layout: A[row][k] where row = lane (0-15), k comes from 8 VGPRs × 2 halves
+  mat_a = []
+  for lane in range(16):
+    for reg in range(8):
+      val = st.vgpr[lane][src0 - 256 + reg] if src0 >= 256 else st.rsgpr(src0 + reg)
+      mat_a.append(_f16(val & 0xffff))
+      mat_a.append(_f16((val >> 16) & 0xffff))
+  # Read matrix B (16x16 f16/bf16) - same layout, B[col][k] where col comes from lane
+  mat_b = []
+  for lane in range(16):
+    for reg in range(8):
+      val = st.vgpr[lane][src1 - 256 + reg] if src1 >= 256 else st.rsgpr(src1 + reg)
+      mat_b.append(_f16(val & 0xffff))
+      mat_b.append(_f16((val >> 16) & 0xffff))
+
   # Read matrix C (16x16 f32) from lanes 0-31, VGPRs src2 to src2+7
-  mat_c = [_f32(st.vgpr[i % 32][src2 - 256 + i // 32] if src2 >= 256 else st.rsgpr(src2 + i // 32)) for i in range(256)]
+  # Layout: element i is at lane (i % 32), VGPR (i // 32) + src2
+  mat_c = []
+  for i in range(256):
+    lane, reg = i % 32, i // 32
+    val = st.vgpr[lane][src2 - 256 + reg] if src2 >= 256 else st.rsgpr(src2 + reg)
+    mat_c.append(_f32(val))
+
   # Compute D = A × B + C (16x16 matrix multiply)
-  mat_d = [sum(mat_a[row*16+k] * mat_b[col*16+k] for k in range(16)) + mat_c[row*16+col] for row in range(16) for col in range(16)]
-  # Write result - f16 packed or f32
+  mat_d = [0.0] * 256
+  for row in range(16):
+    for col in range(16):
+      acc = 0.0
+      for k in range(16):
+        a_val = mat_a[row * 16 + k]
+        b_val = mat_b[col * 16 + k]
+        acc += a_val * b_val
+      mat_d[row * 16 + col] = acc + mat_c[row * 16 + col]
+
+  # Write result matrix D back - same layout as C
   if op == VOP3POp.V_WMMA_F16_16X16X16_F16:
+    # Output is f16, pack 2 values per VGPR
     for i in range(0, 256, 2):
-      st.vgpr[(i//2) % 32][vdst + (i//2)//32] = ((_i16(mat_d[i+1]) & 0xffff) << 16) | (_i16(mat_d[i]) & 0xffff)
+      lane, reg = (i // 2) % 32, (i // 2) // 32
+      lo = _i16(mat_d[i]) & 0xffff
+      hi = _i16(mat_d[i + 1]) & 0xffff
+      st.vgpr[lane][vdst + reg]._val = (hi << 16) | lo
   else:
-    for i in range(256): st.vgpr[i % 32][vdst + i//32] = _i32(mat_d[i])
+    # Output is f32
+    for i in range(256):
+      lane, reg = i % 32, i // 32
+      st.vgpr[lane][vdst + reg]._val = _i32(mat_d[i])
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # MAIN EXECUTION LOOP
 # ═══════════════════════════════════════════════════════════════════════════════
 
+SCALAR_TYPES = {SOP1, SOP2, SOPC, SOPK, SOPP, SMEM}
+VECTOR_TYPES = {VOP1, VOP2, VOP3, VOP3SD, VOPC, FLAT, DS, VOPD, VOP3P}
+
+# Pre-cache compiled functions for fast lookup
+_COMPILED_CACHE: dict | None = None
+def _get_fn(op_cls, op):
+  global _COMPILED_CACHE
+  if _COMPILED_CACHE is None: _COMPILED_CACHE = _get_compiled()
+  return _COMPILED_CACHE.get(op_cls, {}).get(op)
+
+def exec_vector_batch(st: WaveState, inst: Inst, exec_mask: int, n_lanes: int, lds: bytearray | None = None) -> None:
+  """Execute vector instruction for all active lanes at once."""
+  compiled = _get_compiled()
+  inst_type = type(inst)
+  vgpr = st.vgpr
+
+  # Memory ops - still per-lane but inlined
+  if inst_type is FLAT:
+    op, addr_reg, data_reg, vdst, offset, saddr = inst.op, inst.addr, inst.data, inst.vdst, _sext(inst.offset, 13), inst.saddr
+    if op in FLAT_LOAD:
+      cnt, sz, sign = FLAT_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        for i in range(cnt): val = mem_read(addr + i * sz, sz); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in FLAT_STORE:
+      cnt, sz = FLAT_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        for i in range(cnt): mem_write(addr + i * sz, sz, V[data_reg + i]._val & ((1 << (sz * 8)) - 1))
+    elif op in FLAT_D16_LOAD:
+      sz, sign, hi = FLAT_D16_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        val = mem_read(addr, sz)
+        if sign: val = _sext(val, sz * 8) & 0xffff
+        if hi: V[vdst]._val = (V[vdst]._val & 0xffff) | (val << 16)
+        else: V[vdst]._val = (V[vdst]._val & 0xffff0000) | (val & 0xffff)
+    elif op in FLAT_D16_STORE:
+      sz, hi = FLAT_D16_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        val = (V[data_reg]._val >> 16) & 0xffff if hi else V[data_reg]._val & 0xffff
+        mem_write(addr, sz, val & ((1 << (sz * 8)) - 1))
+    else: raise NotImplementedError(f"FLAT op {op}")
+    return
+
+  if inst_type is DS:
+    op, vdst = inst.op, inst.vdst
+    if op in DS_LOAD:
+      cnt, sz, sign = DS_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = (V[inst.addr]._val + inst.offset0) & 0xffff
+        for i in range(cnt): val = int.from_bytes(lds[addr+i*sz:addr+i*sz+sz], 'little'); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in DS_STORE:
+      cnt, sz = DS_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = (V[inst.addr]._val + inst.offset0) & 0xffff
+        for i in range(cnt): lds[addr+i*sz:addr+i*sz+sz] = (V[inst.data0 + i]._val & ((1 << (sz * 8)) - 1)).to_bytes(sz, 'little')
+    else: raise NotImplementedError(f"DS op {op}")
+    return
+
+  # For VOPC, VOP3-encoded VOPC, and VOP3SD, we write per-lane bits to an SGPR.
+  # The pseudocode does D0.u64[laneId] = bit or VCC.u64[laneId] = bit.
+  # To avoid corrupting reads from the same SGPR, use a shared output Reg(0).
+  # Exception: CMPX instructions write to EXEC (not D0/VCC).
+  d0_override, vcc_override = None, None
+  vopc_dst, vop3sd_dst = None, None
+  is_cmpx = False
+  if inst_type is VOPC:
+    op = VOPCOp(inst.op)
+    is_cmpx = 'CMPX' in op.name
+    if not is_cmpx:  # Regular CMP writes to VCC
+      d0_override, vopc_dst = Reg(0), VCC_LO
+    else:  # CMPX writes to EXEC - clear it first, accumulate per-lane
+      st.sgpr[EXEC_LO]._val = 0
+  elif inst_type is VOP3 and inst.op < 256:  # VOP3-encoded VOPC
+    op = VOPCOp(inst.op)
+    is_cmpx = 'CMPX' in op.name
+    if not is_cmpx:  # Regular CMP writes to destination SGPR
+      d0_override, vopc_dst = Reg(0), inst.vdst
+    else:  # CMPX writes to EXEC - clear it first, accumulate per-lane
+      st.sgpr[EXEC_LO]._val = 0
+  if inst_type is VOP3SD:
+    vcc_override, vop3sd_dst = Reg(0), inst.sdst
+
+  # For other vector ops, dispatch to exec_vector per lane (can optimize later)
+  for lane in range(n_lanes):
+    if exec_mask & (1 << lane): exec_vector(st, inst, lane, lds, d0_override, vcc_override)
+
+  # Write accumulated per-lane bit results to destination SGPRs
+  # (CMPX writes directly to EXEC in the pseudocode, so no separate write needed)
+  if vopc_dst is not None: st.sgpr[vopc_dst]._val = d0_override._val
+  if vop3sd_dst is not None: st.sgpr[vop3sd_dst]._val = vcc_override._val
+
 def step_wave(program: Program, st: WaveState, lds: bytearray, n_lanes: int) -> int:
   inst = program.get(st.pc)
   if inst is None: return 1
-  inst_words, st.literal = inst._words, getattr(inst, '_literal', None) or 0
+  inst_words, st.literal, inst_type = inst._words, getattr(inst, '_literal', None) or 0, type(inst)
 
-  if isinstance(inst, (SOP1, SOP2, SOPC, SOPK, SOPP, SMEM)):
+  if inst_type in SCALAR_TYPES:
     delta = exec_scalar(st, inst)
     if delta == -1: return -1  # endpgm
     if delta == -2: st.pc += inst_words; return -2  # barrier
     st.pc += inst_words + delta
   else:
-    # V_READFIRSTLANE/V_READLANE write to SGPR, execute once; others execute per-lane with exec_mask
-    is_readlane = isinstance(inst, (VOP1, VOP3)) and ('READFIRSTLANE' in inst.op_name or 'READLANE' in inst.op_name)
-    exec_mask = 1 if is_readlane else st.exec_mask
-    for lane in range(1 if is_readlane else n_lanes):
-      if exec_mask & (1 << lane): exec_vector(st, inst, lane, lds)
+    # V_READFIRSTLANE_B32 and V_READLANE_B32 write to SGPR, so they should only execute once per wave (lane 0)
+    is_readlane = (inst_type is VOP1 and inst.op == VOP1Op.V_READFIRSTLANE_B32) or \
+                  (inst_type is VOP3 and inst.op in (VOP3Op.V_READFIRSTLANE_B32, VOP3Op.V_READLANE_B32))
+    if is_readlane:
+      exec_vector(st, inst, 0, lds)  # Execute once with lane 0
+    else:
+      exec_vector_batch(st, inst, st.exec_mask, n_lanes, lds)
     st.commit_pends()
     st.pc += inst_words
   return 0
@@ -439,24 +729,31 @@ def exec_workgroup(program: Program, workgroup_id: tuple[int, int, int], local_s
     n_lanes, st = min(WAVE_SIZE, total_threads - wave_start), WaveState()
     st.exec_mask = (1 << n_lanes) - 1
     st.wsgpr64(0, args_ptr)
+    gx, gy, gz = workgroup_id
     # Set workgroup IDs in SGPRs based on USER_SGPR_COUNT and enable flags from COMPUTE_PGM_RSRC2
     sgpr_idx = wg_id_sgpr_base
-    for wg_id, enabled in zip(workgroup_id, wg_id_enables):
-      if enabled: st.sgpr[sgpr_idx] = wg_id; sgpr_idx += 1
-    # Set workitem IDs in VGPR0 using packed method: v0 = (Z << 20) | (Y << 10) | X
+    if wg_id_enables[0]: st.sgpr[sgpr_idx]._val = gx; sgpr_idx += 1
+    if wg_id_enables[1]: st.sgpr[sgpr_idx]._val = gy; sgpr_idx += 1
+    if wg_id_enables[2]: st.sgpr[sgpr_idx]._val = gz
     for i in range(n_lanes):
       tid = wave_start + i
-      st.vgpr[i][0] = ((tid // (lx * ly)) << 20) | (((tid // lx) % ly) << 10) | (tid % lx)
+      st.vgpr[i][0]._val = tid if local_size == (lx, 1, 1) else ((tid // (lx * ly)) << 20) | (((tid // lx) % ly) << 10) | (tid % lx)
     waves.append((st, n_lanes, wave_start))
   has_barrier = any(isinstance(inst, SOPP) and inst.op == SOPPOp.S_BARRIER for inst in program.values())
   for _ in range(2 if has_barrier else 1):
     for st, n_lanes, _ in waves: exec_wave(program, st, lds, n_lanes)
 
 def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c) -> int:
-  program = decode_program((ctypes.c_char * lib_sz).from_address(lib).raw)
+  data = (ctypes.c_char * lib_sz).from_address(lib).raw
+  program = decode_program(data)
   if not program: return -1
-  wg_id_enables = tuple(bool((rsrc2 >> (7+i)) & 1) for i in range(3))
+  # Parse COMPUTE_PGM_RSRC2 for SGPR layout
+  user_sgpr_count = (rsrc2 >> 1) & 0x1f
+  enable_wg_id_x = bool((rsrc2 >> 7) & 1)
+  enable_wg_id_y = bool((rsrc2 >> 8) & 1)
+  enable_wg_id_z = bool((rsrc2 >> 9) & 1)
+  wg_id_enables = (enable_wg_id_x, enable_wg_id_y, enable_wg_id_z)
   for gidz in range(gz):
     for gidy in range(gy):
-      for gidx in range(gx): exec_workgroup(program, (gidx, gidy, gidz), (lx, ly, lz), args_ptr, (rsrc2 >> 1) & 0x1f, wg_id_enables)
+      for gidx in range(gx): exec_workgroup(program, (gidx, gidy, gidz), (lx, ly, lz), args_ptr, user_sgpr_count, wg_id_enables)
   return 0

extra/assembly/amd/pcode.py

@@ -1,53 +1,95 @@
 # DSL for RDNA3 pseudocode - makes pseudocode expressions work directly as Python
-import struct, math
-from extra.assembly.amd.dsl import MASK32, MASK64, _f32, _i32, _sext, _f16, _i16, _f64, _i64
+import struct, math, re
 
 # ═══════════════════════════════════════════════════════════════════════════════
-# HELPER FUNCTIONS
+# HELPER FUNCTIONS (previously in helpers.py)
 # ═══════════════════════════════════════════════════════════════════════════════
 
+def _f32(i): return struct.unpack("<f", struct.pack("<I", i & 0xffffffff))[0]
+def _i32(f):
+  if isinstance(f, int): f = float(f)
+  if math.isnan(f): return 0xffc00000 if math.copysign(1.0, f) < 0 else 0x7fc00000
+  if math.isinf(f): return 0x7f800000 if f > 0 else 0xff800000
+  try: return struct.unpack("<I", struct.pack("<f", f))[0]
+  except (OverflowError, struct.error): return 0x7f800000 if f > 0 else 0xff800000
 def _div(a, b):
   try: return a / b
   except ZeroDivisionError:
     if a == 0.0 or math.isnan(a): return float("nan")
     return math.copysign(float("inf"), a * b) if b == 0.0 else float("inf") if a > 0 else float("-inf")
+def _sext(v, b): return v - (1 << b) if v & (1 << (b - 1)) else v
+def _f16(i): return struct.unpack("<e", struct.pack("<H", i & 0xffff))[0]
+def _i16(f):
+  if math.isnan(f): return 0x7e00
+  if math.isinf(f): return 0x7c00 if f > 0 else 0xfc00
+  try: return struct.unpack("<H", struct.pack("<e", f))[0]
+  except (OverflowError, struct.error): return 0x7c00 if f > 0 else 0xfc00
 def _to_f16_bits(v): return v if isinstance(v, int) else _i16(v)
+def _f64(i): return struct.unpack("<d", struct.pack("<Q", i & 0xffffffffffffffff))[0]
+def _i64(f):
+  if math.isnan(f): return 0x7ff8000000000000
+  if math.isinf(f): return 0x7ff0000000000000 if f > 0 else 0xfff0000000000000
+  try: return struct.unpack("<Q", struct.pack("<d", f))[0]
+  except (OverflowError, struct.error): return 0x7ff0000000000000 if f > 0 else 0xfff0000000000000
 def _isnan(x):
   try: return math.isnan(float(x))
   except (TypeError, ValueError): return False
-def _check_nan_type(x, quiet_bit_expected, default):
-  """Check NaN type by examining quiet bit. Returns default if can't determine."""
+def _isquietnan(x):
+  """Check if x is a quiet NaN. For f32: exponent=255, bit22=1, mantissa!=0"""
   try:
     if not math.isnan(float(x)): return False
+    # Get raw bits from TypedView or similar object with _reg attribute
     if hasattr(x, '_reg') and hasattr(x, '_bits'):
       bits = x._reg._val & ((1 << x._bits) - 1)
-      # NaN format: exponent all 1s, quiet bit, mantissa != 0
-      # f16: exp[14:10]=31, quiet=bit9, mant[8:0]  |  f32: exp[30:23]=255, quiet=bit22, mant[22:0]  |  f64: exp[62:52]=2047, quiet=bit51, mant[51:0]
-      exp_bits, quiet_pos, mant_mask = {16: (0x1f, 9, 0x3ff), 32: (0xff, 22, 0x7fffff), 64: (0x7ff, 51, 0xfffffffffffff)}.get(x._bits, (0,0,0))
-      exp_shift = {16: 10, 32: 23, 64: 52}.get(x._bits, 0)
-      if exp_bits and ((bits >> exp_shift) & exp_bits) == exp_bits and (bits & mant_mask) != 0:
-        return ((bits >> quiet_pos) & 1) == quiet_bit_expected
-    return default
+      if x._bits == 32:
+        return ((bits >> 23) & 0xff) == 255 and ((bits >> 22) & 1) == 1 and (bits & 0x7fffff) != 0
+      if x._bits == 64:
+        return ((bits >> 52) & 0x7ff) == 0x7ff and ((bits >> 51) & 1) == 1 and (bits & 0xfffffffffffff) != 0
+    return True  # Default to quiet NaN if we can't determine bit pattern
+  except (TypeError, ValueError): return False
+def _issignalnan(x):
+  """Check if x is a signaling NaN. For f32: exponent=255, bit22=0, mantissa!=0"""
+  try:
+    if not math.isnan(float(x)): return False
+    # Get raw bits from TypedView or similar object with _reg attribute
+    if hasattr(x, '_reg') and hasattr(x, '_bits'):
+      bits = x._reg._val & ((1 << x._bits) - 1)
+      if x._bits == 32:
+        return ((bits >> 23) & 0xff) == 255 and ((bits >> 22) & 1) == 0 and (bits & 0x7fffff) != 0
+      if x._bits == 64:
+        return ((bits >> 52) & 0x7ff) == 0x7ff and ((bits >> 51) & 1) == 0 and (bits & 0xfffffffffffff) != 0
+    return False  # Default to not signaling if we can't determine bit pattern
   except (TypeError, ValueError): return False
-def _isquietnan(x): return _check_nan_type(x, 1, True)  # quiet NaN has quiet bit = 1
-def _issignalnan(x): return _check_nan_type(x, 0, False)  # signaling NaN has quiet bit = 0
 def _gt_neg_zero(a, b): return (a > b) or (a == 0 and b == 0 and not math.copysign(1, a) < 0 and math.copysign(1, b) < 0)
 def _lt_neg_zero(a, b): return (a < b) or (a == 0 and b == 0 and math.copysign(1, a) < 0 and not math.copysign(1, b) < 0)
 def _fma(a, b, c): return a * b + c
 def _signext(v): return v
-def _fpop(fn): return lambda x: (x := float(x), x if math.isnan(x) or math.isinf(x) else float(fn(x)))[1]
-trunc, floor, ceil = _fpop(math.trunc), _fpop(math.floor), _fpop(math.ceil)
-class _SafeFloat(float):
-  """Float subclass that uses _div for division to handle 0/inf correctly."""
-  def __truediv__(self, o): return _div(float(self), float(o))
-  def __rtruediv__(self, o): return _div(float(o), float(self))
-def sqrt(x): return _SafeFloat(math.sqrt(x)) if x >= 0 else _SafeFloat(float("nan"))
+def trunc(x):
+  x = float(x)
+  return x if math.isnan(x) or math.isinf(x) else float(math.trunc(x))
+def floor(x):
+  x = float(x)
+  return x if math.isnan(x) or math.isinf(x) else float(math.floor(x))
+def ceil(x):
+  x = float(x)
+  return x if math.isnan(x) or math.isinf(x) else float(math.ceil(x))
+def sqrt(x): return math.sqrt(x) if x >= 0 else float("nan")
 def log2(x): return math.log2(x) if x > 0 else (float("-inf") if x == 0 else float("nan"))
 i32_to_f32 = u32_to_f32 = i32_to_f64 = u32_to_f64 = f32_to_f64 = f64_to_f32 = float
-def _f_to_int(f, lo, hi): f = float(f); return 0 if math.isnan(f) else (hi if f >= hi else lo if f <= lo else int(f))
-def f32_to_i32(f): return _f_to_int(f, -2147483648, 2147483647)
-def f32_to_u32(f): return _f_to_int(f, 0, 4294967295)
-f64_to_i32, f64_to_u32 = f32_to_i32, f32_to_u32
+def f32_to_i32(f):
+  f = float(f)
+  if math.isnan(f): return 0
+  if f >= 2147483647: return 2147483647
+  if f <= -2147483648: return -2147483648
+  return int(f)
+def f32_to_u32(f):
+  f = float(f)
+  if math.isnan(f): return 0
+  if f >= 4294967295: return 4294967295
+  if f <= 0: return 0
+  return int(f)
+f64_to_i32 = f32_to_i32
+f64_to_u32 = f32_to_u32
 def f32_to_f16(f):
   f = float(f)
   if math.isnan(f): return 0x7e00  # f16 NaN
@@ -65,32 +107,41 @@ def u4_to_u32(v): return int(v) & 0xf
 def _sign(f): return 1 if math.copysign(1.0, f) < 0 else 0
 def _mantissa_f32(f): return struct.unpack("<I", struct.pack("<f", f))[0] & 0x7fffff if not (math.isinf(f) or math.isnan(f)) else 0
 def _ldexp(m, e): return math.ldexp(m, e)
-def isEven(x):
-  x = float(x)
-  if math.isinf(x) or math.isnan(x): return False
-  return int(x) % 2 == 0
+def isEven(x): return int(x) % 2 == 0
 def fract(x): return x - math.floor(x)
 PI = math.pi
-def _trig(fn, x):
-  # V_SIN/COS_F32: hardware does frac on input cycles before computing
+def sin(x):
+  # V_SIN_F32: pseudocode does sin(input * 2π), but hardware does frac on the input first
+  # So sin(1.0 * 2π) should be sin(frac(1.0) * 2π) = sin(0) = 0
   if math.isinf(x) or math.isnan(x): return float("nan")
-  frac_cycles = fract(x / (2 * math.pi))
-  return fn(frac_cycles * 2 * math.pi)
-def sin(x): return _trig(math.sin, x)
-def cos(x): return _trig(math.cos, x)
+  # The input x is already multiplied by 2π in the pseudocode, so we need to
+  # extract the fractional cycle: frac(x / 2π) * 2π
+  cycles = x / (2 * math.pi)
+  frac_cycles = cycles - math.floor(cycles)
+  return math.sin(frac_cycles * 2 * math.pi)
+def cos(x):
+  # V_COS_F32: same as sin, hardware does frac on input cycles
+  if math.isinf(x) or math.isnan(x): return float("nan")
+  cycles = x / (2 * math.pi)
+  frac_cycles = cycles - math.floor(cycles)
+  return math.cos(frac_cycles * 2 * math.pi)
 def pow(a, b):
   try: return a ** b
   except OverflowError: return float("inf") if b > 0 else 0.0
-def _brev(v, bits): return int(bin(v & ((1 << bits) - 1))[2:].zfill(bits)[::-1], 2)
-def _brev32(v): return _brev(v, 32)
-def _brev64(v): return _brev(v, 64)
-def _ctz(v, bits):
-  v, n = int(v) & ((1 << bits) - 1), 0
-  if v == 0: return bits
+def _brev32(v): return int(bin(v & 0xffffffff)[2:].zfill(32)[::-1], 2)
+def _brev64(v): return int(bin(v & 0xffffffffffffffff)[2:].zfill(64)[::-1], 2)
+def _ctz32(v):
+  v = int(v) & 0xffffffff
+  if v == 0: return 32
+  n = 0
+  while (v & 1) == 0: v >>= 1; n += 1
+  return n
+def _ctz64(v):
+  v = int(v) & 0xffffffffffffffff
+  if v == 0: return 64
+  n = 0
   while (v & 1) == 0: v >>= 1; n += 1
   return n
-def _ctz32(v): return _ctz(v, 32)
-def _ctz64(v): return _ctz(v, 64)
 def _exponent(f):
   # Handle TypedView (f16/f32/f64) to get correct exponent for that type
   if hasattr(f, '_bits') and hasattr(f, '_float') and f._float:
@@ -114,21 +165,34 @@ def _is_denorm_f64(f):
   if math.isinf(f) or math.isnan(f) or f == 0.0: return False
   bits = struct.unpack("<Q", struct.pack("<d", float(f)))[0]
   return (bits >> 52) & 0x7ff == 0
-def v_min_f32(a, b): return a if math.isnan(b) else b if math.isnan(a) else (a if _lt_neg_zero(a, b) else b)
-def v_max_f32(a, b): return a if math.isnan(b) else b if math.isnan(a) else (a if _gt_neg_zero(a, b) else b)
-v_min_f16, v_max_f16 = v_min_f32, v_max_f32
-v_min_i32, v_max_i32 = min, max
-v_min_i16, v_max_i16 = min, max
-def v_min_u32(a, b): return min(a & MASK32, b & MASK32)
-def v_max_u32(a, b): return max(a & MASK32, b & MASK32)
+def v_min_f32(a, b):
+  if math.isnan(b): return a
+  if math.isnan(a): return b
+  return a if _lt_neg_zero(a, b) else b
+def v_max_f32(a, b):
+  if math.isnan(b): return a
+  if math.isnan(a): return b
+  return a if _gt_neg_zero(a, b) else b
+def v_min_i32(a, b): return min(a, b)
+def v_max_i32(a, b): return max(a, b)
+def v_min_u32(a, b): return min(a & 0xffffffff, b & 0xffffffff)
+def v_max_u32(a, b): return max(a & 0xffffffff, b & 0xffffffff)
+v_min_f16 = v_min_f32
+v_max_f16 = v_max_f32
+v_min_i16 = v_min_i32
+v_max_i16 = v_max_i32
 def v_min_u16(a, b): return min(a & 0xffff, b & 0xffff)
 def v_max_u16(a, b): return max(a & 0xffff, b & 0xffff)
 def v_min3_f32(a, b, c): return v_min_f32(v_min_f32(a, b), c)
 def v_max3_f32(a, b, c): return v_max_f32(v_max_f32(a, b), c)
-v_min3_f16, v_max3_f16 = v_min3_f32, v_max3_f32
-v_min3_i32, v_max3_i32, v_min3_i16, v_max3_i16 = min, max, min, max
-def v_min3_u32(a, b, c): return min(a & MASK32, b & MASK32, c & MASK32)
-def v_max3_u32(a, b, c): return max(a & MASK32, b & MASK32, c & MASK32)
+def v_min3_i32(a, b, c): return min(a, b, c)
+def v_max3_i32(a, b, c): return max(a, b, c)
+def v_min3_u32(a, b, c): return min(a & 0xffffffff, b & 0xffffffff, c & 0xffffffff)
+def v_max3_u32(a, b, c): return max(a & 0xffffffff, b & 0xffffffff, c & 0xffffffff)
+v_min3_f16 = v_min3_f32
+v_max3_f16 = v_max3_f32
+v_min3_i16 = v_min3_i32
+v_max3_i16 = v_max3_i32
 def v_min3_u16(a, b, c): return min(a & 0xffff, b & 0xffff, c & 0xffff)
 def v_max3_u16(a, b, c): return max(a & 0xffff, b & 0xffff, c & 0xffff)
 def ABSDIFF(a, b): return abs(int(a) - int(b))
@@ -197,7 +261,7 @@ def f32_to_u8(f): return max(0, min(255, int(f))) if not math.isnan(f) else 0
 def mantissa(f):
   if f == 0.0 or math.isinf(f) or math.isnan(f): return f
   m, _ = math.frexp(f)
-  return m  # AMD V_FREXP_MANT returns mantissa in [0.5, 1.0) range
+  return math.copysign(m * 2.0, f)
 def signext_from_bit(val, bit):
   bit = int(bit)
   if bit == 0: return 0
@@ -212,13 +276,12 @@ def signext_from_bit(val, bit):
 
 __all__ = [
   # Classes
-  'Reg', 'SliceProxy', 'TypedView',
+  'Reg', 'SliceProxy', 'TypedView', 'ExecContext', 'compile_pseudocode',
   # Pack functions
   '_pack', '_pack32', 'pack', 'pack32',
   # Constants
   'WAVE32', 'WAVE64', 'MASK32', 'MASK64', 'WAVE_MODE', 'DENORM', 'OVERFLOW_F32', 'UNDERFLOW_F32',
   'OVERFLOW_F64', 'UNDERFLOW_F64', 'MAX_FLOAT_F32', 'ROUND_MODE', 'cvtToQuietNAN', 'DST', 'INF', 'PI',
-  'TWO_OVER_PI_1201',
   # Aliases for pseudocode
   's_ff1_i32_b32', 's_ff1_i32_b64', 'GT_NEG_ZERO', 'LT_NEG_ZERO',
   'isNAN', 'isQuietNAN', 'isSignalNAN', 'fma', 'ldexp', 'sign', 'exponent', 'F', 'signext',
@@ -259,7 +322,7 @@ def F(x):
   if isinstance(x, int): return _f32(x)  # int -> interpret as f32 bits
   if isinstance(x, TypedView): return x  # preserve TypedView for bit-pattern checks
   return float(x)  # already a float or float-like
-signext = lambda x: int(x)  # sign-extend to full width - already handled by Python's arbitrary precision ints
+signext = lambda x: x
 pack = lambda hi, lo: ((int(hi) & 0xffff) << 16) | (int(lo) & 0xffff)
 pack32 = lambda hi, lo: ((int(hi) & 0xffffffff) << 32) | (int(lo) & 0xffffffff)
 _pack, _pack32 = pack, pack32  # Aliases for internal use
@@ -277,14 +340,12 @@ class _Inf:
   f16 = f32 = f64 = float('inf')
   def __neg__(self): return _NegInf()
   def __pos__(self): return self
-  def __float__(self): return float('inf')
   def __eq__(self, other): return float(other) == float('inf') if not isinstance(other, _NegInf) else False
   def __req__(self, other): return self.__eq__(other)
 class _NegInf:
   f16 = f32 = f64 = float('-inf')
   def __neg__(self): return _Inf()
   def __pos__(self): return self
-  def __float__(self): return float('-inf')
   def __eq__(self, other): return float(other) == float('-inf') if not isinstance(other, _Inf) else False
   def __req__(self, other): return self.__eq__(other)
 INF = _Inf()
@@ -298,30 +359,7 @@ ROUND_MODE = _RoundMode()
 def cvtToQuietNAN(x): return float('nan')
 DST = None  # Placeholder, will be set in context
 
-# 2/PI with 1201 bits of precision for V_TRIG_PREOP_F64
-# Computed as: int((2/pi) * 2^1201) - this is the fractional part of 2/pi scaled to integer
-# The MSB (bit 1200) corresponds to 2^0 position in the fraction 0.b1200 b1199 ... b1 b0
-_TWO_OVER_PI_1201_RAW = 0x0145f306dc9c882a53f84eafa3ea69bb81b6c52b3278872083fca2c757bd778ac36e48dc74849ba5c00c925dd413a32439fc3bd63962534e7dd1046bea5d768909d338e04d68befc827323ac7306a673e93908bf177bf250763ff12fffbc0b301fde5e2316b414da3eda6cfd9e4f96136e9e8c7ecd3cbfd45aea4f758fd7cbe2f67a0e73ef14a525d4d7f6bf623f1aba10ac06608df8f6
-
-class _BigInt:
-  """Wrapper for large integers that supports bit slicing [high:low]."""
-  __slots__ = ('_val',)
-  def __init__(self, val): self._val = val
-  def __getitem__(self, key):
-    if isinstance(key, slice):
-      high, low = key.start, key.stop
-      if high < low: high, low = low, high  # Handle reversed slice
-      mask = (1 << (high - low + 1)) - 1
-      return (self._val >> low) & mask
-    return (self._val >> key) & 1
-  def __int__(self): return self._val
-  def __index__(self): return self._val
-  def __lshift__(self, n): return self._val << int(n)
-  def __rshift__(self, n): return self._val >> int(n)
-  def __and__(self, n): return self._val & int(n)
-  def __or__(self, n): return self._val | int(n)
-
-TWO_OVER_PI_1201 = _BigInt(_TWO_OVER_PI_1201_RAW)
+MASK32, MASK64 = 0xffffffff, 0xffffffffffffffff
 
 class _WaveMode:
   IEEE = False
@@ -462,17 +500,6 @@ class TypedView:
 
   def __bool__(s): return bool(int(s))
 
-  # Allow chained type access like jump_addr.i64 when jump_addr is already a TypedView
-  # These just return self or convert appropriately
-  @property
-  def i64(s): return s if s._bits == 64 and s._signed else int(s)
-  @property
-  def u64(s): return s if s._bits == 64 and not s._signed else int(s) & MASK64
-  @property
-  def i32(s): return s if s._bits == 32 and s._signed else _sext(int(s) & MASK32, 32)
-  @property
-  def u32(s): return s if s._bits == 32 and not s._signed else int(s) & MASK32
-
 class Reg:
   """GPU register: D0.f32 = S0.f32 + S1.f32 just works."""
   __slots__ = ('_val',)
@@ -496,7 +523,6 @@ class Reg:
   bf16 = property(lambda s: TypedView(s, 16, is_float=True, is_bf16=True), lambda s, v: setattr(s, '_val', (s._val & 0xffff0000) | ((v if isinstance(v, int) else _ibf16(float(v))) & 0xffff)))
   u8 = property(lambda s: TypedView(s, 8))
   i8 = property(lambda s: TypedView(s, 8, signed=True))
-  u1 = property(lambda s: TypedView(s, 1))  # single bit
 
   def __getitem__(s, key):
     if isinstance(key, slice): return SliceProxy(s, int(key.start), int(key.stop))
@@ -538,5 +564,482 @@ class Reg:
   def __ge__(s, o): return s._val >= int(o)
   def __eq__(s, o): return s._val == int(o)
   def __ne__(s, o): return s._val != int(o)
+  def __format__(s, spec): return format(s._val, spec)
+  def __repr__(s): return f"Reg(0x{s._val:x})"
 
+# ═══════════════════════════════════════════════════════════════════════════════
+# COMPILER: pseudocode -> Python (minimal transforms)
+# ═══════════════════════════════════════════════════════════════════════════════
 
+def compile_pseudocode(pseudocode: str) -> str:
+  """Compile pseudocode to Python. Transforms are minimal - most syntax just works."""
+  # Join continuation lines (lines ending with || or && or open paren)
+  raw_lines = pseudocode.strip().split('\n')
+  joined_lines: list[str] = []
+  for line in raw_lines:
+    line = line.strip()
+    if joined_lines and (joined_lines[-1].rstrip().endswith(('||', '&&', '(', ',')) or
+                         (joined_lines[-1].count('(') > joined_lines[-1].count(')'))):
+      joined_lines[-1] = joined_lines[-1].rstrip() + ' ' + line
+    else:
+      joined_lines.append(line)
+
+  lines = []
+  indent, need_pass = 0, False
+  for line in joined_lines:
+    line = line.strip()
+    if not line or line.startswith('//'): continue
+
+    # Control flow - only need pass before outdent (endif/endfor/else/elsif)
+    if line.startswith('if '):
+      lines.append('  ' * indent + f"if {_expr(line[3:].rstrip(' then'))}:")
+      indent += 1
+      need_pass = True
+    elif line.startswith('elsif '):
+      if need_pass: lines.append('  ' * indent + "pass")
+      indent -= 1
+      lines.append('  ' * indent + f"elif {_expr(line[6:].rstrip(' then'))}:")
+      indent += 1
+      need_pass = True
+    elif line == 'else':
+      if need_pass: lines.append('  ' * indent + "pass")
+      indent -= 1
+      lines.append('  ' * indent + "else:")
+      indent += 1
+      need_pass = True
+    elif line.startswith('endif'):
+      if need_pass: lines.append('  ' * indent + "pass")
+      indent -= 1
+      need_pass = False
+    elif line.startswith('endfor'):
+      if need_pass: lines.append('  ' * indent + "pass")
+      indent -= 1
+      need_pass = False
+    elif line.startswith('declare '):
+      pass
+    elif m := re.match(r'for (\w+) in (.+?)\s*:\s*(.+?) do', line):
+      start, end = _expr(m[2].strip()), _expr(m[3].strip())
+      lines.append('  ' * indent + f"for {m[1]} in range({start}, int({end})+1):")
+      indent += 1
+      need_pass = True
+    elif '=' in line and not line.startswith('=='):
+      need_pass = False
+      line = line.rstrip(';')
+      # Handle tuple unpacking: { D1.u1, D0.u64 } = expr
+      if m := re.match(r'\{\s*D1\.[ui]1\s*,\s*D0\.[ui]64\s*\}\s*=\s*(.+)', line):
+        rhs = _expr(m[1])
+        lines.append('  ' * indent + f"_full = {rhs}")
+        lines.append('  ' * indent + f"D0.u64 = int(_full) & 0xffffffffffffffff")
+        lines.append('  ' * indent + f"D1 = Reg((int(_full) >> 64) & 1)")
+      # Compound assignment
+      elif any(op in line for op in ('+=', '-=', '*=', '/=', '|=', '&=', '^=')):
+        for op in ('+=', '-=', '*=', '/=', '|=', '&=', '^='):
+          if op in line:
+            lhs, rhs = line.split(op, 1)
+            lines.append('  ' * indent + f"{lhs.strip()} {op} {_expr(rhs.strip())}")
+            break
+      else:
+        lhs, rhs = line.split('=', 1)
+        lines.append('  ' * indent + _assign(lhs.strip(), _expr(rhs.strip())))
+  # If we ended with a control statement that needs a body, add pass
+  if need_pass: lines.append('  ' * indent + "pass")
+  return '\n'.join(lines)
+
+def _assign(lhs: str, rhs: str) -> str:
+  """Generate assignment. Bare tmp/SCC/etc get wrapped in Reg(). For params (SCC, VCC, EXEC, D0), modify in place."""
+  # Parameters passed to function - modify in place using .b32 setter (or .b64 for 64-bit types)
+  if lhs in ('SCC', 'VCC', 'EXEC', 'D0'):
+    return f"{lhs}.b32 = {rhs}"
+  # Local variables - create new Reg
+  if lhs in ('tmp', 'D1', 'saveexec'):
+    return f"{lhs} = Reg({rhs})"
+  return f"{lhs} = {rhs}"
+
+def _expr(e: str) -> str:
+  """Expression transform: minimal - just fix syntax differences."""
+  e = e.strip()
+  e = e.replace('&&', ' and ').replace('||', ' or ').replace('<>', ' != ')
+  e = re.sub(r'!([^=])', r' not \1', e)
+
+  # Pack: { hi, lo } -> _pack(hi, lo)
+  e = re.sub(r'\{\s*(\w+\.u32)\s*,\s*(\w+\.u32)\s*\}', r'_pack32(\1, \2)', e)
+  def pack(m):
+    hi, lo = _expr(m[1].strip()), _expr(m[2].strip())
+    return f'_pack({hi}, {lo})'
+  e = re.sub(r'\{\s*([^,{}]+)\s*,\s*([^,{}]+)\s*\}', pack, e)
+
+  # Literals: 1'0U -> 0, 32'I(x) -> (x), B(x) -> (x)
+  e = re.sub(r"\d+'([0-9a-fA-Fx]+)[UuFf]*", r'\1', e)
+  e = re.sub(r"\d+'[FIBU]\(", "(", e)
+  e = re.sub(r'\bB\(', '(', e)  # Bare B( without digit prefix
+  e = re.sub(r'([0-9a-fA-Fx])ULL\b', r'\1', e)
+  e = re.sub(r'([0-9a-fA-Fx])LL\b', r'\1', e)
+  e = re.sub(r'([0-9a-fA-Fx])U\b', r'\1', e)
+  e = re.sub(r'(\d\.?\d*)F\b', r'\1', e)
+  # Remove redundant type suffix after lane access: VCC.u64[laneId].u64 -> VCC.u64[laneId]
+  e = re.sub(r'(\[laneId\])\.[uib]\d+', r'\1', e)
+
+  # Constants - INF is defined as an object supporting .f32/.f64 access
+  e = e.replace('+INF', 'INF').replace('-INF', '(-INF)')
+  e = re.sub(r'NAN\.f\d+', 'float("nan")', e)
+
+  # Verilog bit slice syntax: [start +: width] -> extract width bits starting at start
+  # Convert to Python slice: [start + width - 1 : start]
+  def convert_verilog_slice(m):
+    start, width = m.group(1).strip(), m.group(2).strip()
+    # Convert to high:low slice format
+    return f'[({start}) + ({width}) - 1 : ({start})]'
+  e = re.sub(r'\[([^:\[\]]+)\s*\+:\s*([^:\[\]]+)\]', convert_verilog_slice, e)
+
+  # Recursively process bracket contents to handle nested ternaries like S1.u32[x ? a : b]
+  def process_brackets(s):
+    result, i = [], 0
+    while i < len(s):
+      if s[i] == '[':
+        # Find matching ]
+        depth, start = 1, i + 1
+        j = start
+        while j < len(s) and depth > 0:
+          if s[j] == '[': depth += 1
+          elif s[j] == ']': depth -= 1
+          j += 1
+        inner = _expr(s[start:j-1])  # Recursively process bracket content
+        result.append('[' + inner + ']')
+        i = j
+      else:
+        result.append(s[i])
+        i += 1
+    return ''.join(result)
+  e = process_brackets(e)
+
+  # Ternary: a ? b : c -> (b if a else c)
+  while '?' in e:
+    depth, bracket, q = 0, 0, -1
+    for i, c in enumerate(e):
+      if c == '(': depth += 1
+      elif c == ')': depth -= 1
+      elif c == '[': bracket += 1
+      elif c == ']': bracket -= 1
+      elif c == '?' and depth == 0 and bracket == 0: q = i; break
+    if q < 0: break
+    depth, bracket, col = 0, 0, -1
+    for i in range(q + 1, len(e)):
+      if e[i] == '(': depth += 1
+      elif e[i] == ')': depth -= 1
+      elif e[i] == '[': bracket += 1
+      elif e[i] == ']': bracket -= 1
+      elif e[i] == ':' and depth == 0 and bracket == 0: col = i; break
+    if col < 0: break
+    cond, t, f = e[:q].strip(), e[q+1:col].strip(), e[col+1:].strip()
+    e = f'(({t}) if ({cond}) else ({f}))'
+  return e
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EXECUTION CONTEXT
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class ExecContext:
+  """Context for running compiled pseudocode."""
+  def __init__(self, s0=0, s1=0, s2=0, d0=0, scc=0, vcc=0, lane=0, exec_mask=MASK32, literal=0, vgprs=None, src0_idx=0, vdst_idx=0):
+    self.S0, self.S1, self.S2 = Reg(s0), Reg(s1), Reg(s2)
+    self.D0, self.D1 = Reg(d0), Reg(0)
+    self.SCC, self.VCC, self.EXEC = Reg(scc), Reg(vcc), Reg(exec_mask)
+    self.tmp, self.saveexec = Reg(0), Reg(exec_mask)
+    self.lane, self.laneId, self.literal = lane, lane, literal
+    self.SIMM16, self.SIMM32 = Reg(literal), Reg(literal)
+    self.VGPR = vgprs if vgprs is not None else {}
+    self.SRC0, self.VDST = Reg(src0_idx), Reg(vdst_idx)
+
+  def run(self, code: str):
+    """Execute compiled code."""
+    # Start with module globals (helpers, aliases), then add instance-specific bindings
+    ns = dict(globals())
+    ns.update({
+      'S0': self.S0, 'S1': self.S1, 'S2': self.S2, 'D0': self.D0, 'D1': self.D1,
+      'SCC': self.SCC, 'VCC': self.VCC, 'EXEC': self.EXEC,
+      'EXEC_LO': SliceProxy(self.EXEC, 31, 0), 'EXEC_HI': SliceProxy(self.EXEC, 63, 32),
+      'tmp': self.tmp, 'saveexec': self.saveexec,
+      'lane': self.lane, 'laneId': self.laneId, 'literal': self.literal,
+      'SIMM16': self.SIMM16, 'SIMM32': self.SIMM32,
+      'VGPR': self.VGPR, 'SRC0': self.SRC0, 'VDST': self.VDST,
+    })
+    exec(code, ns)
+    # Sync rebinds: if register was reassigned to new Reg or value, copy it back
+    def _sync(ctx_reg, ns_val):
+      if isinstance(ns_val, Reg): ctx_reg._val = ns_val._val
+      else: ctx_reg._val = int(ns_val) & MASK64
+    if ns.get('SCC') is not self.SCC: _sync(self.SCC, ns['SCC'])
+    if ns.get('VCC') is not self.VCC: _sync(self.VCC, ns['VCC'])
+    if ns.get('EXEC') is not self.EXEC: _sync(self.EXEC, ns['EXEC'])
+    if ns.get('D0') is not self.D0: _sync(self.D0, ns['D0'])
+    if ns.get('D1') is not self.D1: _sync(self.D1, ns['D1'])
+    if ns.get('tmp') is not self.tmp: _sync(self.tmp, ns['tmp'])
+    if ns.get('saveexec') is not self.saveexec: _sync(self.saveexec, ns['saveexec'])
+
+  def result(self) -> dict:
+    return {"d0": self.D0._val, "scc": self.SCC._val & 1}
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# PDF EXTRACTION AND CODE GENERATION
+# ═══════════════════════════════════════════════════════════════════════════════
+
+from extra.assembly.amd.dsl import PDF_URLS
+INST_PATTERN = re.compile(r'^([SV]_[A-Z0-9_]+)\s+(\d+)\s*$', re.M)
+
+# Patterns that can't be handled by the DSL (require special handling in emu.py)
+UNSUPPORTED = ['SGPR[', 'V_SWAP', 'eval ', 'FATAL_HALT', 'HW_REGISTERS',
+               'PC =', 'PC=', 'PC+', '= PC', 'vscnt', 'vmcnt', 'expcnt', 'lgkmcnt',
+               'CVT_OFF_TABLE', 'ThreadMask',
+               'S1[i', 'C.i32', 'S[i]', 'in[', '2.0 / PI',
+               'if n.', 'DST.u32', 'addrd = DST', 'addr = DST']  # Malformed pseudocode from PDF
+
+def extract_pseudocode(text: str) -> str | None:
+  """Extract pseudocode from an instruction description snippet."""
+  lines, result, depth = text.split('\n'), [], 0
+  for line in lines:
+    s = line.strip()
+    if not s: continue
+    if re.match(r'^\d+ of \d+$', s): continue
+    if re.match(r'^\d+\.\d+\..*Instructions', s): continue
+    # Skip document headers (RDNA or CDNA)
+    if s.startswith('"RDNA') or s.startswith('AMD ') or s.startswith('CDNA'): continue
+    if s.startswith('Notes') or s.startswith('Functional examples'): break
+    if s.startswith('if '): depth += 1
+    elif s.startswith('endif'): depth = max(0, depth - 1)
+    if s.endswith('.') and not any(p in s for p in ['D0', 'D1', 'S0', 'S1', 'S2', 'SCC', 'VCC', 'tmp', '=']): continue
+    if re.match(r'^[a-z].*\.$', s) and '=' not in s: continue
+    is_code = (
+      any(p in s for p in ['D0.', 'D1.', 'S0.', 'S1.', 'S2.', 'SCC =', 'SCC ?', 'VCC', 'EXEC', 'tmp =', 'tmp[', 'lane =']) or
+      any(p in s for p in ['D0[', 'D1[', 'S0[', 'S1[', 'S2[']) or
+      s.startswith(('if ', 'else', 'elsif', 'endif', 'declare ', 'for ', 'endfor', '//')) or
+      re.match(r'^[a-z_]+\s*=', s) or re.match(r'^[a-z_]+\[', s) or (depth > 0 and '=' in s)
+    )
+    if is_code: result.append(s)
+  return '\n'.join(result) if result else None
+
+def _get_op_enums(arch: str) -> list:
+  """Dynamically load op enums from the arch-specific autogen module."""
+  import importlib
+  autogen = importlib.import_module(f"extra.assembly.amd.autogen.{arch}")
+  # Deterministic order: common enums first, then arch-specific
+  enums = []
+  for name in ['SOP1Op', 'SOP2Op', 'SOPCOp', 'SOPKOp', 'SOPPOp', 'VOP1Op', 'VOP2Op', 'VOP3Op', 'VOP3SDOp', 'VOP3POp', 'VOPCOp', 'VOP3AOp', 'VOP3BOp']:
+    if hasattr(autogen, name): enums.append(getattr(autogen, name))
+  return enums
+
+def _parse_pseudocode_from_single_pdf(url: str, defined_ops: dict, OP_ENUMS: list) -> dict:
+  """Parse pseudocode from a single PDF."""
+  import pdfplumber
+  from tinygrad.helpers import fetch
+
+  pdf = pdfplumber.open(fetch(url))
+  total_pages = len(pdf.pages)
+
+  page_cache = {}
+  def get_page_text(i):
+    if i not in page_cache: page_cache[i] = pdf.pages[i].extract_text() or ''
+    return page_cache[i]
+
+  # Find the "Instructions" chapter - typically 10-40% through the document
+  instr_start = None
+  for i in range(int(total_pages * 0.1), int(total_pages * 0.5)):
+    if re.search(r'Chapter \d+\.\s+Instructions\b', get_page_text(i)):
+      instr_start = i
+      break
+  if instr_start is None: instr_start = total_pages // 3  # fallback
+
+  # Find end - stop at "Microcode Formats" chapter (typically 60-70% through)
+  instr_end = total_pages
+  search_starts = [int(total_pages * 0.6), int(total_pages * 0.5), instr_start]
+  for start in search_starts:
+    for i in range(start, min(start + 100, total_pages)):
+      if re.search(r'Chapter \d+\.\s+Microcode Formats', get_page_text(i)):
+        instr_end = i
+        break
+    if instr_end < total_pages: break
+
+  # Extract remaining pages (some already cached from chapter search)
+  all_text = '\n'.join(get_page_text(i) for i in range(instr_start, instr_end))
+  matches = list(INST_PATTERN.finditer(all_text))
+  instructions: dict = {cls: {} for cls in OP_ENUMS}
+
+  for i, match in enumerate(matches):
+    name, opcode = match.group(1), int(match.group(2))
+    key = (name, opcode)
+    if key not in defined_ops: continue
+    start = match.end()
+    end = matches[i + 1].start() if i + 1 < len(matches) else start + 2000
+    snippet = all_text[start:end].strip()
+    if (pseudocode := extract_pseudocode(snippet)):
+      # Assign to all enums that have this op (e.g., both VOPCOp and VOP3AOp)
+      for enum_cls, enum_val in defined_ops[key]:
+        instructions[enum_cls][enum_val] = pseudocode
+
+  return instructions
+
+def parse_pseudocode_from_pdf(arch: str = "rdna3") -> dict:
+  """Parse pseudocode from PDF(s) for all ops. Returns {enum_cls: {op: pseudocode}}."""
+  OP_ENUMS = _get_op_enums(arch)
+  # Build a dict from (name, opcode) -> list of (enum_cls, op) tuples
+  # Multiple enums can have the same op (e.g., VOPCOp and VOP3AOp both have V_CMP_* ops)
+  defined_ops: dict[tuple, list] = {}
+  for enum_cls in OP_ENUMS:
+    for op in enum_cls:
+      if op.name.startswith(('S_', 'V_')): defined_ops.setdefault((op.name, op.value), []).append((enum_cls, op))
+
+  urls = PDF_URLS[arch]
+  if isinstance(urls, str): urls = [urls]
+
+  # Parse all PDFs and merge (union of pseudocode)
+  # Reverse order so newer PDFs (RDNA3.5, CDNA4) take priority
+  instructions: dict = {cls: {} for cls in OP_ENUMS}
+  for url in reversed(urls):
+    result = _parse_pseudocode_from_single_pdf(url, defined_ops, OP_ENUMS)
+    for cls, ops in result.items():
+      for op, pseudocode in ops.items():
+        if op in instructions[cls]:
+          if instructions[cls][op] != pseudocode:
+            print(f"  Ignoring {op.name} from older PDF:")
+            print(f"    new: {instructions[cls][op]!r}")
+            print(f"    old: {pseudocode!r}")
+        else:
+          instructions[cls][op] = pseudocode
+
+  return instructions
+
+def generate_gen_pcode(output_path: str = "extra/assembly/amd/autogen/rdna3/gen_pcode.py", arch: str = "rdna3"):
+  """Generate gen_pcode.py - compiled pseudocode functions for the emulator."""
+  from pathlib import Path
+
+  OP_ENUMS = _get_op_enums(arch)
+
+  print("Parsing pseudocode from PDF...")
+  by_cls = parse_pseudocode_from_pdf(arch)
+
+  total_found, total_ops = 0, 0
+  for enum_cls in OP_ENUMS:
+    total = sum(1 for op in enum_cls if op.name.startswith(('S_', 'V_')))
+    found = len(by_cls.get(enum_cls, {}))
+    total_found += found
+    total_ops += total
+    print(f"{enum_cls.__name__}: {found}/{total} ({100*found//total if total else 0}%)")
+  print(f"Total: {total_found}/{total_ops} ({100*total_found//total_ops}%)")
+
+  print("\nCompiling to pseudocode functions...")
+  # Build dynamic import line based on available enums
+  enum_names = [e.__name__ for e in OP_ENUMS]
+  lines = [f'''# autogenerated by pcode.py - do not edit
+# to regenerate: python -m extra.assembly.amd.pcode --arch {arch}
+# ruff: noqa: E501,F405,F403
+# mypy: ignore-errors
+from extra.assembly.amd.autogen.{arch} import {", ".join(enum_names)}
+from extra.assembly.amd.pcode import *
+''']
+
+  compiled_count, skipped_count = 0, 0
+
+  for enum_cls in OP_ENUMS:
+    cls_name = enum_cls.__name__
+    pseudocode_dict = by_cls.get(enum_cls, {})
+    if not pseudocode_dict: continue
+
+    fn_entries = []
+    for op, pc in pseudocode_dict.items():
+      if any(p in pc for p in UNSUPPORTED):
+        skipped_count += 1
+        continue
+
+      try:
+        code = compile_pseudocode(pc)
+        # CLZ/CTZ: The PDF pseudocode searches for the first 1 bit but doesn't break.
+        # Hardware stops at first match. SOP1 uses tmp=i, VOP1/VOP3 use D0.i32=i
+        if 'CLZ' in op.name or 'CTZ' in op.name:
+          code = code.replace('tmp = Reg(i)', 'tmp._val = i; break')
+          code = code.replace('D0.i32 = i', 'D0.i32 = i; break')
+        # Detect flags for result handling
+        is_64 = any(p in pc for p in ['D0.u64', 'D0.b64', 'D0.f64', 'D0.i64', 'D1.u64', 'D1.b64', 'D1.f64', 'D1.i64'])
+        has_d1 = '{ D1' in pc
+        if has_d1: is_64 = True
+        is_cmp = cls_name == 'VOPCOp' and 'D0.u64[laneId]' in pc
+        is_cmpx = cls_name == 'VOPCOp' and 'EXEC.u64[laneId]' in pc  # V_CMPX writes to EXEC per-lane
+        # V_DIV_SCALE passes through S0 if no branch taken
+        is_div_scale = 'DIV_SCALE' in op.name
+        # VOP3SD instructions that write VCC per-lane (either via VCC.u64[laneId] or by setting VCC = 0/1)
+        has_sdst = cls_name == 'VOP3SDOp' and ('VCC.u64[laneId]' in pc or is_div_scale)
+
+        # Generate function that takes Reg objects directly - modifies D0 in place
+        fn_name = f"_{cls_name}_{op.name}"
+        lines.append(f"def {fn_name}(S0, S1, S2, D0, SCC, VCC, laneId, EXEC, SIMM16, VGPR, SRC0, VDST):")
+        # Add original pseudocode as comment
+        for pc_line in pc.split('\n'):
+          lines.append(f"  # {pc_line}")
+        # Only create extra Reg objects for registers that need fresh state
+        combined = code + pc
+        # D1 and tmp/saveexec need to be created fresh
+        if 'D1' in combined: lines.append("  D1 = Reg(0)")
+        if 'tmp' in combined: lines.append("  tmp = Reg(0)")
+        if 'saveexec' in combined: lines.append("  saveexec = Reg(EXEC._val)")
+        if 'SIMM32' in combined: lines.append("  SIMM32 = SIMM16")
+        if 'EXEC_LO' in combined: lines.append("  EXEC_LO = SliceProxy(EXEC, 31, 0)")
+        if 'EXEC_HI' in combined: lines.append("  EXEC_HI = SliceProxy(EXEC, 63, 32)")
+        # For DIV_SCALE, D0 starts with S0's value
+        if is_div_scale: lines.append("  D0._val = S0._val")
+        # Add compiled pseudocode
+        lines.append("  # --- compiled pseudocode ---")
+        has_code = False
+        for line in code.split('\n'):
+          if line.strip():
+            lines.append(f"  {line}")
+            has_code = True
+        lines.append("  # --- end pseudocode ---")
+        # All Reg objects (D0, SCC, VCC, EXEC) are modified in place
+        # The emulator determines 64-bit ops from the opcode name
+        if not has_code:
+          lines.append("  pass")
+        lines.append("")
+
+        fn_entries.append((op, fn_name))
+        compiled_count += 1
+      except Exception as e:
+        print(f"  Warning: Failed to compile {op.name}: {e}")
+        skipped_count += 1
+
+    if fn_entries:
+      lines.append(f'{cls_name}_FUNCTIONS = {{')
+      for op, fn_name in fn_entries:
+        lines.append(f"  {cls_name}.{op.name}: {fn_name},")
+      lines.append('}')
+      lines.append('')
+
+  # Add manually implemented V_WRITELANE_B32 (not in PDF pseudocode, requires special vgpr_write handling)
+  # Only add for architectures that have VOP3Op (RDNA) not VOP3AOp/VOP3BOp (CDNA)
+  if 'VOP3Op' in enum_names:
+    lines.append('''
+# V_WRITELANE_B32: Write scalar to specific lane's VGPR (not in PDF pseudocode)
+def _VOP3Op_V_WRITELANE_B32(S0, S1, S2, D0, SCC, VCC, laneId, EXEC, SIMM16, VGPR, SRC0, VDST):
+  wr_lane = S1._val & 0x1f  # lane select (5 bits for wave32)
+  return {'vgpr_write': (wr_lane, VDST._val, S0._val & 0xffffffff)}
+VOP3Op_FUNCTIONS[VOP3Op.V_WRITELANE_B32] = _VOP3Op_V_WRITELANE_B32
+''')
+
+  lines.append('COMPILED_FUNCTIONS = {')
+  for enum_cls in OP_ENUMS:
+    cls_name = enum_cls.__name__
+    if by_cls.get(enum_cls): lines.append(f'  {cls_name}: {cls_name}_FUNCTIONS,')
+  lines.append('}')
+  lines.append('')
+  lines.append('def get_compiled_functions(): return COMPILED_FUNCTIONS')
+
+  Path(output_path).write_text('\n'.join(lines))
+  print(f"\nGenerated {output_path}: {compiled_count} compiled, {skipped_count} skipped")
+
+if __name__ == "__main__":
+  import argparse
+  parser = argparse.ArgumentParser(description="Generate pseudocode functions from AMD ISA PDF")
+  parser.add_argument("--arch", choices=list(PDF_URLS.keys()) + ["all"], default="rdna3", help="Target architecture (default: rdna3)")
+  args = parser.parse_args()
+  if args.arch == "all":
+    for arch in PDF_URLS.keys():
+      generate_gen_pcode(output_path=f"extra/assembly/amd/autogen/{arch}/gen_pcode.py", arch=arch)
+  else:
+    generate_gen_pcode(output_path=f"extra/assembly/amd/autogen/{args.arch}/gen_pcode.py", arch=args.arch)

extra/assembly/amd/pdf.py

@@ -1,670 +0,0 @@
-# Generate AMD ISA autogen files from PDF documentation
-# Combines format/enum generation (previously in dsl.py) and pseudocode compilation (previously in pcode.py)
-# Usage: python -m extra.assembly.amd.pdf [--arch rdna3|rdna4|cdna|all]
-import re, functools
-from pathlib import Path
-from concurrent.futures import ProcessPoolExecutor
-
-PDF_URLS = {
-  "rdna3": "https://docs.amd.com/api/khub/documents/UVVZM22UN7tMUeiW_4ShTQ/content",
-  "rdna4": "https://docs.amd.com/api/khub/documents/uQpkEvk3pv~kfAb2x~j4uw/content",
-  "cdna": ["https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-mi300-cdna3-instruction-set-architecture.pdf",
-           "https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf"],
-}
-
-# Field type mappings and ordering
-FIELD_TYPES = {'SSRC0': 'SSrc', 'SSRC1': 'SSrc', 'SOFFSET': 'SSrc', 'SADDR': 'SSrc', 'SRC0': 'Src', 'SRC1': 'Src', 'SRC2': 'Src',
-  'SDST': 'SGPRField', 'SBASE': 'SGPRField', 'SDATA': 'SGPRField', 'SRSRC': 'SGPRField', 'VDST': 'VGPRField', 'VSRC1': 'VGPRField',
-  'VDATA': 'VGPRField', 'VADDR': 'VGPRField', 'ADDR': 'VGPRField', 'DATA': 'VGPRField', 'DATA0': 'VGPRField', 'DATA1': 'VGPRField',
-  'SIMM16': 'SImm', 'OFFSET': 'Imm', 'OPX': 'VOPDOp', 'OPY': 'VOPDOp', 'SRCX0': 'Src', 'SRCY0': 'Src',
-  'VSRCX1': 'VGPRField', 'VSRCY1': 'VGPRField', 'VDSTX': 'VGPRField', 'VDSTY': 'VDSTYEnc'}
-FIELD_ORDER = {
-  'SOP2': ['op', 'sdst', 'ssrc0', 'ssrc1'], 'SOP1': ['op', 'sdst', 'ssrc0'], 'SOPC': ['op', 'ssrc0', 'ssrc1'],
-  'SOPK': ['op', 'sdst', 'simm16'], 'SOPP': ['op', 'simm16'], 'VOP1': ['op', 'vdst', 'src0'], 'VOPC': ['op', 'src0', 'vsrc1'],
-  'VOP2': ['op', 'vdst', 'src0', 'vsrc1'], 'VOP3SD': ['op', 'vdst', 'sdst', 'src0', 'src1', 'src2', 'clmp'],
-  'SMEM': ['op', 'sdata', 'sbase', 'soffset', 'offset', 'glc', 'dlc'], 'DS': ['op', 'vdst', 'addr', 'data0', 'data1'],
-  'VOP3': ['op', 'vdst', 'src0', 'src1', 'src2', 'omod', 'neg', 'abs', 'clmp', 'opsel'],
-  'VOP3P': ['op', 'vdst', 'src0', 'src1', 'src2', 'neg', 'neg_hi', 'opsel', 'opsel_hi', 'clmp'],
-  'FLAT': ['op', 'vdst', 'addr', 'data', 'saddr', 'offset', 'seg', 'dlc', 'glc', 'slc'],
-  'MUBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
-  'MTBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'format', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
-  'MIMG': ['op', 'vdata', 'vaddr', 'srsrc', 'ssamp', 'dmask', 'dim', 'unrm', 'dlc', 'glc', 'slc'],
-  'EXP': ['en', 'target', 'vsrc0', 'vsrc1', 'vsrc2', 'vsrc3', 'done', 'row'],
-  'VINTERP': ['op', 'vdst', 'src0', 'src1', 'src2', 'waitexp', 'clmp', 'opsel', 'neg'],
-  'VOPD': ['opx', 'opy', 'vdstx', 'vdsty', 'srcx0', 'vsrcx1', 'srcy0', 'vsrcy1'],
-  'LDSDIR': ['op', 'vdst', 'attr', 'attr_chan', 'wait_va']}
-SRC_EXTRAS = {233: 'DPP8', 234: 'DPP8FI', 250: 'DPP16', 251: 'VCCZ', 252: 'EXECZ', 254: 'LDS_DIRECT'}
-FLOAT_MAP = {'0.5': 'POS_HALF', '-0.5': 'NEG_HALF', '1.0': 'POS_ONE', '-1.0': 'NEG_ONE', '2.0': 'POS_TWO', '-2.0': 'NEG_TWO',
-  '4.0': 'POS_FOUR', '-4.0': 'NEG_FOUR', '1/(2*PI)': 'INV_2PI', '0': 'ZERO'}
-INST_PATTERN = re.compile(r'^([SVD]S?_[A-Z0-9_]+)\s+(\d+)\s*$', re.M)
-
-# Patterns that can't be handled by the DSL (require special handling in emu.py)
-UNSUPPORTED = ['SGPR[', 'V_SWAP', 'eval ', 'FATAL_HALT', 'HW_REGISTERS',
-               'vscnt', 'vmcnt', 'expcnt', 'lgkmcnt',
-               'CVT_OFF_TABLE', 'ThreadMask',
-               'S1[i', 'C.i32',
-               'if n.', 'DST.u32', 'addrd = DST', 'addr = DST',
-               'BARRIER_STATE', 'ReallocVgprs',
-               'GPR_IDX', 'VSKIP', 'specified in', 'TTBL',
-               'fp6', 'bf6']  # Malformed pseudocode from PDF
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# COMPILER: pseudocode -> Python (minimal transforms)
-# ═══════════════════════════════════════════════════════════════════════════════
-
-def compile_pseudocode(pseudocode: str) -> str:
-  """Compile pseudocode to Python. Transforms are minimal - most syntax just works."""
-  pseudocode = re.sub(r'\bpass\b', 'pass_', pseudocode)  # 'pass' is Python keyword
-  raw_lines = pseudocode.strip().split('\n')
-  joined_lines: list[str] = []
-  for line in raw_lines:
-    line = line.strip()
-    if joined_lines and (joined_lines[-1].rstrip().endswith(('||', '&&', '(', ',')) or
-                         (joined_lines[-1].count('(') > joined_lines[-1].count(')'))):
-      joined_lines[-1] = joined_lines[-1].rstrip() + ' ' + line
-    else:
-      joined_lines.append(line)
-
-  lines = []
-  indent, need_pass, in_first_match_loop = 0, False, False
-  declared_arrays: dict[str, int] = {}  # Track declared arrays: name -> size
-  for line in joined_lines:
-    line = line.strip()
-    if not line or line.startswith('//'): continue
-    if line.startswith('if '):
-      lines.append('  ' * indent + f"if {_expr(line[3:].rstrip(' then'), declared_arrays)}:")
-      indent += 1
-      need_pass = True
-    elif line.startswith('elsif '):
-      if need_pass: lines.append('  ' * indent + "pass")
-      indent -= 1
-      lines.append('  ' * indent + f"elif {_expr(line[6:].rstrip(' then'), declared_arrays)}:")
-      indent += 1
-      need_pass = True
-    elif line == 'else':
-      if need_pass: lines.append('  ' * indent + "pass")
-      indent -= 1
-      lines.append('  ' * indent + "else:")
-      indent += 1
-      need_pass = True
-    elif line.startswith('endif'):
-      if need_pass: lines.append('  ' * indent + "pass")
-      indent -= 1
-      need_pass = False
-    elif line.startswith('endfor'):
-      if need_pass: lines.append('  ' * indent + "pass")
-      indent -= 1
-      need_pass, in_first_match_loop = False, False
-    elif m := re.match(r'declare\s+(\w+)\s*:\s*\d+\'[FBU]\[(\d+)\]', line):
-      # Handle array declarations: declare in : 32'F[3] or declare S : 32'B[3]
-      arr_name, arr_size = m[1], int(m[2])
-      declared_arrays[arr_name] = arr_size
-      py_name = f"{arr_name}_" if arr_name == 'in' else arr_name  # 'in' is Python keyword
-      if arr_name == 'S':
-        lines.append('  ' * indent + f"{py_name} = [S0, S1, S2]")  # Map to source registers
-      else:
-        lines.append('  ' * indent + f"{py_name} = [Reg(0) for _ in range({arr_size})]")
-    elif line.startswith('declare '):
-      pass  # Ignore other declare statements
-    elif m := re.match(r'for (\w+) in (.+?)\s*:\s*(.+?) do', line):
-      start, end = _expr(m[2].strip(), declared_arrays), _expr(m[3].strip(), declared_arrays)
-      lines.append('  ' * indent + f"for {m[1]} in range({start}, int({end})+1):")
-      indent += 1
-      need_pass, in_first_match_loop = True, True
-    elif '=' in line and not line.startswith('=='):
-      need_pass = False
-      line = line.rstrip(';')
-      if m := re.match(r'\{\s*D1\.[ui]1\s*,\s*D0\.[ui]64\s*\}\s*=\s*(.+)', line):
-        rhs = _expr(m[1], declared_arrays)
-        lines.append('  ' * indent + f"_full = {rhs}")
-        lines.append('  ' * indent + f"D0.u64 = int(_full) & 0xffffffffffffffff")
-        lines.append('  ' * indent + f"D1 = Reg((int(_full) >> 64) & 1)")
-      elif any(op in line for op in ('+=', '-=', '*=', '/=', '|=', '&=', '^=')):
-        for op in ('+=', '-=', '*=', '/=', '|=', '&=', '^='):
-          if op in line:
-            lhs, rhs = line.split(op, 1)
-            lhs_s = _expr(lhs.strip(), declared_arrays)  # Transform LHS too for array access
-            lines.append('  ' * indent + f"{lhs_s} {op} {_expr(rhs.strip(), declared_arrays)}")
-            break
-      else:
-        lhs, rhs = line.split('=', 1)
-        lhs_s, rhs_s = lhs.strip(), rhs.strip()
-        lhs_t = _expr(lhs_s, declared_arrays)  # Transform LHS for array access
-        stmt = _assign(lhs_t, _expr(rhs_s, declared_arrays), declared_arrays)
-        if in_first_match_loop and rhs_s == 'i' and (lhs_s == 'tmp' or lhs_s == 'D0.i32'):
-          stmt += "; break"
-        lines.append('  ' * indent + stmt)
-  if need_pass: lines.append('  ' * indent + "pass")
-  return '\n'.join(lines)
-
-def _assign(lhs: str, rhs: str, declared_arrays: dict[str, int] | None = None) -> str:
-  # Check for array element assignment: in_[i] should not wrap in Reg()
-  if declared_arrays and re.match(r'\w+_?\[\w+\]', lhs):
-    return f"{lhs} = {rhs}"
-  if lhs in ('tmp', 'SCC', 'VCC', 'EXEC', 'D0', 'D1', 'saveexec', 'PC'):
-    return f"{lhs} = Reg({rhs})"
-  return f"{lhs} = {rhs}"
-
-def _expr(e: str, declared_arrays: dict[str, int] | None = None) -> str:
-  e = e.strip()
-  # Handle OPSEL_HI.u3[i] and OPSEL.u3[i] - bit extraction from opsel fields
-  e = re.sub(r'(OPSEL(?:_HI)?)\.u\d+\[(\w+)\]', r'((\1 >> \2) & 1)', e)
-  # Rename 'in' to 'in_' to avoid Python keyword conflict
-  e = re.sub(r'\bin\[', 'in_[', e)
-  e = e.replace('&&', ' and ').replace('||', ' or ').replace('<>', ' != ')
-  e = re.sub(r'!([^=])', r' not \1', e)
-  e = re.sub(r'\{\s*(\w+\.u32)\s*,\s*(\w+\.u32)\s*\}', r'_pack32(\1, \2)', e)
-  def pack(m):
-    hi, lo = _expr(m[1].strip(), declared_arrays), _expr(m[2].strip(), declared_arrays)
-    return f'_pack({hi}, {lo})'
-  e = re.sub(r'\{\s*([^,{}]+)\s*,\s*([^,{}]+)\s*\}', pack, e)
-  e = re.sub(r"1201'B\(2\.0\s*/\s*PI\)", "TWO_OVER_PI_1201", e)
-  e = re.sub(r"\d+'([0-9a-fA-Fx]+)[UuFf]*", r'\1', e)
-  e = re.sub(r"\d+'[FIBU]\(", "(", e)
-  e = re.sub(r'\bB\(', '(', e)
-  e = re.sub(r'([0-9a-fA-Fx])ULL\b', r'\1', e)
-  e = re.sub(r'([0-9a-fA-Fx])LL\b', r'\1', e)
-  e = re.sub(r'([0-9a-fA-Fx])U\b', r'\1', e)
-  e = re.sub(r'(\d\.?\d*)F\b', r'\1', e)
-  e = re.sub(r'(\[laneId\])\.[uib]\d+', r'\1', e)
-  e = e.replace('+INF', 'INF').replace('-INF', '(-INF)')
-  e = re.sub(r'NAN\.f\d+', 'float("nan")', e)
-  def convert_verilog_slice(m):
-    start, width = m.group(1).strip(), m.group(2).strip()
-    return f'[({start}) + ({width}) - 1 : ({start})]'
-  e = re.sub(r'\[([^:\[\]]+)\s*\+:\s*([^:\[\]]+)\]', convert_verilog_slice, e)
-  def process_brackets(s):
-    result, i = [], 0
-    while i < len(s):
-      if s[i] == '[':
-        depth, start = 1, i + 1
-        j = start
-        while j < len(s) and depth > 0:
-          if s[j] == '[': depth += 1
-          elif s[j] == ']': depth -= 1
-          j += 1
-        inner = _expr(s[start:j-1], declared_arrays)
-        result.append('[' + inner + ']')
-        i = j
-      else:
-        result.append(s[i])
-        i += 1
-    return ''.join(result)
-  e = process_brackets(e)
-  while '?' in e:
-    depth, bracket, q = 0, 0, -1
-    for i, c in enumerate(e):
-      if c == '(': depth += 1
-      elif c == ')': depth -= 1
-      elif c == '[': bracket += 1
-      elif c == ']': bracket -= 1
-      elif c == '?' and depth == 0 and bracket == 0: q = i; break
-    if q < 0: break
-    depth, bracket, col = 0, 0, -1
-    for i in range(q + 1, len(e)):
-      if e[i] == '(': depth += 1
-      elif e[i] == ')': depth -= 1
-      elif e[i] == '[': bracket += 1
-      elif e[i] == ']': bracket -= 1
-      elif e[i] == ':' and depth == 0 and bracket == 0: col = i; break
-    if col < 0: break
-    cond, t, f = e[:q].strip(), e[q+1:col].strip(), e[col+1:].strip()
-    e = f'(({t}) if ({cond}) else ({f}))'
-  return e
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# PDF PARSING WITH PAGE CACHING
-# ═══════════════════════════════════════════════════════════════════════════════
-
-class CachedPDF:
-  """PDF wrapper with page text/table caching for faster repeated access."""
-  def __init__(self, pdf):
-    self._pdf, self._text_cache, self._table_cache = pdf, {}, {}
-  def __len__(self): return len(self._pdf.pages)
-  def text(self, i):
-    if i not in self._text_cache: self._text_cache[i] = self._pdf.pages[i].extract_text() or ''
-    return self._text_cache[i]
-  def tables(self, i):
-    if i not in self._table_cache: self._table_cache[i] = [t.extract() for t in self._pdf.pages[i].find_tables()]
-    return self._table_cache[i]
-
-def _parse_bits(s: str) -> tuple[int, int] | None:
-  return (int(m.group(1)), int(m.group(2) or m.group(1))) if (m := re.match(r'\[(\d+)(?::(\d+))?\]', s)) else None
-
-def _parse_fields_table(table: list, fmt: str, enums: set[str]) -> list[tuple]:
-  fields = []
-  for row in table[1:]:
-    if not row or not row[0]: continue
-    name, bits_str = row[0].split('\n')[0].strip(), (row[1] or '').split('\n')[0].strip()
-    if not (bits := _parse_bits(bits_str)): continue
-    enc_val, hi, lo = None, bits[0], bits[1]
-    if name == 'ENCODING' and row[2]:
-      if m := re.search(r"(?:'b|Must be:\s*)([01_]+)", row[2]):
-        enc_bits = m.group(1).replace('_', '')
-        enc_val, declared_width, actual_width = int(enc_bits, 2), hi - lo + 1, len(enc_bits)
-        if actual_width > declared_width: lo = hi - actual_width + 1
-    ftype = f"{fmt}Op" if name == 'OP' and f"{fmt}Op" in enums else FIELD_TYPES.get(name.upper())
-    fields.append((name, hi, lo, enc_val, ftype))
-  return fields
-
-def _parse_single_pdf(url: str):
-  """Parse a single PDF and return (formats, enums, src_enum, doc_name, instructions)."""
-  import pdfplumber
-  from tinygrad.helpers import fetch
-
-  pdf = CachedPDF(pdfplumber.open(fetch(url)))
-  total_pages = len(pdf)
-
-  # Auto-detect document type
-  first_page = pdf.text(0)
-  is_cdna4, is_cdna3 = 'CDNA4' in first_page or 'CDNA 4' in first_page, 'CDNA3' in first_page or 'MI300' in first_page
-  is_cdna, is_rdna4 = is_cdna3 or is_cdna4, 'RDNA4' in first_page or 'RDNA 4' in first_page
-  is_rdna35, is_rdna3 = 'RDNA3.5' in first_page or 'RDNA 3.5' in first_page, 'RDNA3' in first_page and 'RDNA3.5' not in first_page
-  doc_name = "CDNA4" if is_cdna4 else "CDNA3" if is_cdna3 else "RDNA4" if is_rdna4 else "RDNA3.5" if is_rdna35 else "RDNA3" if is_rdna3 else "Unknown"
-
-  # Find Microcode Formats section (for formats/enums)
-  microcode_start = next((i for i in range(int(total_pages * 0.2), total_pages)
-                          if re.search(r'\d+\.\d+\.\d+\.\s+SOP2\b|Chapter \d+\.\s+Microcode Formats', pdf.text(i))), int(total_pages * 0.9))
-  # Find Instructions section (for pseudocode)
-  instr_start = next((i for i in range(int(total_pages * 0.1), int(total_pages * 0.5))
-                      if re.search(r'Chapter \d+\.\s+Instructions\b', pdf.text(i))), total_pages // 3)
-  instr_end = next((i for start in [int(total_pages * 0.6), int(total_pages * 0.5), instr_start]
-                    for i in range(start, min(start + 100, total_pages))
-                    if re.search(r'Chapter \d+\.\s+Microcode Formats', pdf.text(i))), total_pages)
-
-  # Parse src enum from SSRC encoding table
-  src_enum = dict(SRC_EXTRAS)
-  for i in range(microcode_start, min(microcode_start + 10, total_pages)):
-    text = pdf.text(i)
-    if 'SSRC0' in text and 'VCC_LO' in text:
-      for m in re.finditer(r'^(\d+)\s+(\S+)', text, re.M):
-        val, name = int(m.group(1)), m.group(2).rstrip('.:')
-        if name in FLOAT_MAP: src_enum[val] = FLOAT_MAP[name]
-        elif re.match(r'^[A-Z][A-Z0-9_]*$', name): src_enum[val] = name
-      break
-
-  # Parse opcode tables
-  full_text = '\n'.join(pdf.text(i) for i in range(microcode_start, min(microcode_start + 50, total_pages)))
-  enums: dict[str, dict[int, str]] = {}
-  for m in re.finditer(r'Table \d+\. (\w+) Opcodes(.*?)(?=Table \d+\.|\n\d+\.\d+\.\d+\.\s+\w+\s*\nDescription|$)', full_text, re.S):
-    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+([A-Z][A-Z0-9_]+)', m.group(2))}:
-      enums[m.group(1) + "Op"] = ops
-  if vopd_m := re.search(r'Table \d+\. VOPD Y-Opcodes\n(.*?)(?=Table \d+\.|15\.\d)', full_text, re.S):
-    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+(V_DUAL_\w+)', vopd_m.group(1))}:
-      enums["VOPDOp"] = ops
-  enum_names = set(enums.keys())
-
-  # Parse instruction formats
-  def is_fields_table(t): return t and len(t) > 1 and t[0] and 'Field' in str(t[0][0] or '')
-  def has_encoding(fields): return any(f[0] == 'ENCODING' for f in fields)
-  def has_header_before_fields(text): return (pos := text.find('Field Name')) != -1 and bool(re.search(r'\d+\.\d+\.\d+\.\s+\w+\s*\n', text[:pos]))
-
-  format_headers = []
-  for i in range(50):
-    if microcode_start + i >= total_pages: break
-    text = pdf.text(microcode_start + i)
-    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n?Description', text): format_headers.append((m.group(1), i, m.start()))
-    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n', text):
-      fmt_name = m.group(1)
-      if is_cdna and fmt_name.isupper() and len(fmt_name) >= 2: format_headers.append((fmt_name, i, m.start()))
-      elif m.start() > len(text) - 200 and 'Description' not in text[m.end():] and i + 1 < 50:
-        next_text = pdf.text(microcode_start + i + 1).lstrip()
-        if next_text.startswith('Description') or (next_text.startswith('"RDNA') and 'Description' in next_text[:200]):
-          format_headers.append((fmt_name, i, m.start()))
-
-  formats: dict[str, list] = {}
-  for fmt_name, rel_idx, header_pos in format_headers:
-    if fmt_name in formats: continue
-    page_idx = microcode_start + rel_idx
-    text = pdf.text(page_idx)
-    field_pos = text.find('Field Name', header_pos)
-    fields = None
-    for offset in range(3):
-      if page_idx + offset >= total_pages: break
-      if offset > 0 and has_header_before_fields(pdf.text(page_idx + offset)): break
-      for t in pdf.tables(page_idx + offset) if offset > 0 or field_pos > header_pos else []:
-        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)) and has_encoding(f): fields = f; break
-      if fields: break
-    if not fields and field_pos > header_pos:
-      for t in pdf.tables(page_idx):
-        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)): fields = f; break
-    if not fields: continue
-    field_names = {f[0] for f in fields}
-    for pg_offset in range(1, 3):
-      if page_idx + pg_offset >= total_pages or has_header_before_fields(pdf.text(page_idx + pg_offset)): break
-      for t in pdf.tables(page_idx + pg_offset):
-        if is_fields_table(t) and (extra := _parse_fields_table(t, fmt_name, enum_names)) and not has_encoding(extra):
-          for ef in extra:
-            if ef[0] not in field_names: fields.append(ef); field_names.add(ef[0])
-          break
-    formats[fmt_name] = fields
-
-  # Fix known PDF errors
-  if 'SMEM' in formats:
-    formats['SMEM'] = [(n, 13 if n == 'DLC' else 14 if n == 'GLC' else h, 13 if n == 'DLC' else 14 if n == 'GLC' else l, e, t)
-                       for n, h, l, e, t in formats['SMEM']]
-  if doc_name in ('RDNA3', 'RDNA3.5'):
-    if 'SOPPOp' in enums: assert 8 not in enums['SOPPOp']; enums['SOPPOp'][8] = 'S_WAITCNT_DEPCTR'
-    if 'DSOp' in enums:
-      for k, v in {24: 'DS_GWS_SEMA_RELEASE_ALL', 25: 'DS_GWS_INIT', 26: 'DS_GWS_SEMA_V', 27: 'DS_GWS_SEMA_BR', 28: 'DS_GWS_SEMA_P', 29: 'DS_GWS_BARRIER'}.items():
-        assert k not in enums['DSOp']; enums['DSOp'][k] = v
-    if 'FLATOp' in enums:
-      for k, v in {40: 'GLOBAL_LOAD_ADDTID_B32', 41: 'GLOBAL_STORE_ADDTID_B32', 55: 'FLAT_ATOMIC_CSUB_U32'}.items():
-        assert k not in enums['FLATOp']; enums['FLATOp'][k] = v
-
-  # Extract pseudocode for instructions
-  all_text = '\n'.join(pdf.text(i) for i in range(instr_start, instr_end))
-  matches = list(INST_PATTERN.finditer(all_text))
-  raw_pseudocode: dict[tuple[str, int], str] = {}
-  for i, match in enumerate(matches):
-    name, opcode = match.group(1), int(match.group(2))
-    start, end = match.end(), matches[i + 1].start() if i + 1 < len(matches) else match.end() + 2000
-    snippet = all_text[start:end].strip()
-    if pseudocode := _extract_pseudocode(snippet): raw_pseudocode[(name, opcode)] = pseudocode
-
-  return {"formats": formats, "enums": enums, "src_enum": src_enum, "doc_name": doc_name, "pseudocode": raw_pseudocode, "is_cdna": is_cdna}
-
-def _extract_pseudocode(text: str) -> str | None:
-  """Extract pseudocode from an instruction description snippet."""
-  lines, result, depth, in_lambda = text.split('\n'), [], 0, 0
-  for line in lines:
-    s = line.strip()
-    if not s or re.match(r'^\d+ of \d+$', s) or re.match(r'^\d+\.\d+\..*Instructions', s): continue
-    if s.startswith(('Notes', 'Functional examples')): break
-    if s.startswith(('"RDNA', 'AMD ', 'CDNA')): continue
-    if '= lambda(' in s: in_lambda += 1; continue
-    if in_lambda > 0:
-      if s.endswith(');'): in_lambda -= 1
-      continue
-    if s.startswith('if '): depth += 1
-    elif s.startswith('endif'): depth = max(0, depth - 1)
-    if s.endswith('.') and not any(p in s for p in ['D0', 'D1', 'S0', 'S1', 'S2', 'SCC', 'VCC', 'tmp', '=']): continue
-    if re.match(r'^[a-z].*\.$', s) and '=' not in s: continue
-    is_code = (any(p in s for p in ['D0.', 'D1.', 'S0.', 'S1.', 'S2.', 'SCC =', 'SCC ?', 'VCC', 'EXEC', 'tmp =', 'tmp[', 'lane =', 'PC =',
-                                    'D0[', 'D1[', 'S0[', 'S1[', 'S2[', 'MEM[', 'RETURN_DATA', 'DATA.', 'DATA0', 'DATA1', 'ADDR']) or
-               s.startswith(('if ', 'else', 'elsif', 'endif', 'declare ', 'for ', 'endfor', '//')) or
-               re.match(r'^[a-z_]+\s*=', s) or re.match(r'^[a-z_]+\[', s) or (depth > 0 and '=' in s))
-    if is_code: result.append(s)
-  return '\n'.join(result) if result else None
-
-def _merge_results(results: list[dict]) -> dict:
-  """Merge multiple PDF parse results into a superset."""
-  merged = {"formats": {}, "enums": {}, "src_enum": dict(SRC_EXTRAS), "doc_names": [], "pseudocode": {}, "is_cdna": False}
-  for r in results:
-    merged["doc_names"].append(r["doc_name"])
-    merged["is_cdna"] = merged["is_cdna"] or r["is_cdna"]
-    for val, name in r["src_enum"].items():
-      if val in merged["src_enum"]: assert merged["src_enum"][val] == name
-      else: merged["src_enum"][val] = name
-    for enum_name, ops in r["enums"].items():
-      if enum_name not in merged["enums"]: merged["enums"][enum_name] = {}
-      for val, name in ops.items():
-        if val in merged["enums"][enum_name]: assert merged["enums"][enum_name][val] == name
-        else: merged["enums"][enum_name][val] = name
-    for fmt_name, fields in r["formats"].items():
-      if fmt_name not in merged["formats"]: merged["formats"][fmt_name] = list(fields)
-      else:
-        existing = {f[0]: (f[1], f[2]) for f in merged["formats"][fmt_name]}
-        for f in fields:
-          if f[0] in existing: assert existing[f[0]] == (f[1], f[2])
-          else: merged["formats"][fmt_name].append(f)
-    for key, pc in r["pseudocode"].items():
-      if key not in merged["pseudocode"]: merged["pseudocode"][key] = pc
-  return merged
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# CODE GENERATION
-# ═══════════════════════════════════════════════════════════════════════════════
-
-def _generate_enum_py(enums, src_enum, doc_name) -> str:
-  """Generate enum.py content (just enums, no dsl.py dependency)."""
-  def enum_lines(name, items): return [f"class {name}(IntEnum):"] + [f"  {n} = {v}" for v, n in sorted(items.items())] + [""]
-  lines = [f"# autogenerated from AMD {doc_name} ISA PDF by pdf.py - do not edit", "from enum import IntEnum", ""]
-  lines += enum_lines("SrcEnum", src_enum) + sum([enum_lines(n, ops) for n, ops in sorted(enums.items())], [])
-  return '\n'.join(lines)
-
-def _generate_ins_py(formats, enums, src_enum, doc_name) -> str:
-  """Generate ins.py content (instruction formats and helpers, imports dsl.py and enum.py)."""
-  def field_key(f, order): return order.index(f[0].lower()) if f[0].lower() in order else 1000
-  lines = [f"# autogenerated from AMD {doc_name} ISA PDF by pdf.py - do not edit",
-           "# ruff: noqa: F401,F403", "from typing import Annotated",
-           "from extra.assembly.amd.dsl import bits, BitField, Inst32, Inst64, SGPR, VGPR, TTMP as TTMP, s as s, v as v, ttmp as ttmp, SSrc, Src, SImm, Imm, VDSTYEnc, SGPRField, VGPRField",
-           "from extra.assembly.amd.autogen.{arch}.enum import *",
-           "import functools", ""]
-  format_defaults = {'VOP3P': {'opsel_hi': 3, 'opsel_hi2': 1}}
-  lines.append("# instruction formats")
-  for fmt_name, fields in sorted(formats.items()):
-    base = "Inst64" if max(f[1] for f in fields) > 31 or fmt_name == 'VOP3SD' else "Inst32"
-    order = FIELD_ORDER.get(fmt_name, [])
-    lines.append(f"class {fmt_name}({base}):")
-    if enc := next((f for f in fields if f[0] == 'ENCODING'), None):
-      lines.append(f"  encoding = bits[{enc[1]}:{enc[2]}] == 0b{enc[3]:b}" if enc[1] != enc[2] else f"  encoding = bits[{enc[1]}] == {enc[3]}")
-    if defaults := format_defaults.get(fmt_name): lines.append(f"  _defaults = {defaults}")
-    for name, hi, lo, _, ftype in sorted([f for f in fields if f[0] != 'ENCODING'], key=lambda f: field_key(f, order)):
-      ann = f":Annotated[BitField, {ftype}]" if ftype and ftype.endswith('Op') else f":{ftype}" if ftype else ""
-      lines.append(f"  {name.lower()}{ann} = bits[{hi}]" if hi == lo else f"  {name.lower()}{ann} = bits[{hi}:{lo}]")
-    lines.append("")
-  lines.append("# instruction helpers")
-  for cls_name, ops in sorted(enums.items()):
-    fmt = cls_name[:-2]
-    for op_val, name in sorted(ops.items()):
-      seg = {"GLOBAL": ", seg=2", "SCRATCH": ", seg=1"}.get(fmt, "")
-      tgt = {"GLOBAL": "FLAT, GLOBALOp", "SCRATCH": "FLAT, SCRATCHOp"}.get(fmt, f"{fmt}, {cls_name}")
-      if fmt in formats or fmt in ("GLOBAL", "SCRATCH"):
-        suffix = "_e32" if fmt in ("VOP1", "VOP2", "VOPC") else "_e64" if fmt == "VOP3" and op_val < 512 else ""
-        if name in ('V_FMAMK_F32', 'V_FMAMK_F16'):
-          lines.append(f"def {name.lower()}{suffix}(vdst, src0, K, vsrc1): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
-        elif name in ('V_FMAAK_F32', 'V_FMAAK_F16'):
-          lines.append(f"def {name.lower()}{suffix}(vdst, src0, vsrc1, K): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
-        else: lines.append(f"{name.lower()}{suffix} = functools.partial({tgt}.{name}{seg})")
-  src_names = {name for _, name in src_enum.items()}
-  lines += [""] + [f"{name} = SrcEnum.{name}" for _, name in sorted(src_enum.items()) if name not in {'DPP8', 'DPP16'}]
-  if "NULL" in src_names: lines.append("OFF = NULL\n")
-  return '\n'.join(lines)
-
-def _generate_gen_pcode_py(enums, pseudocode, arch) -> str:
-  """Generate gen_pcode.py content (compiled pseudocode functions)."""
-  # Get op enums for this arch (import from .ins which re-exports from .enum)
-  import importlib
-  autogen = importlib.import_module(f"extra.assembly.amd.autogen.{arch}.ins")
-  OP_ENUMS = [getattr(autogen, name) for name in ['SOP1Op', 'SOP2Op', 'SOPCOp', 'SOPKOp', 'SOPPOp', 'VOP1Op', 'VOP2Op', 'VOP3Op', 'VOP3SDOp', 'VOP3POp', 'VOPCOp', 'VOP3AOp', 'VOP3BOp', 'DSOp'] if hasattr(autogen, name)]
-
-  # Build defined ops mapping
-  defined_ops: dict[tuple, list] = {}
-  for enum_cls in OP_ENUMS:
-    for op in enum_cls:
-      if op.name.startswith(('S_', 'V_', 'DS_')): defined_ops.setdefault((op.name, op.value), []).append((enum_cls, op))
-
-  enum_names = [e.__name__ for e in OP_ENUMS]
-  lines = [f'''# autogenerated by pdf.py - do not edit
-# to regenerate: python -m extra.assembly.amd.pdf --arch {arch}
-# ruff: noqa: E501,F405,F403
-# mypy: ignore-errors
-from extra.assembly.amd.autogen.{arch}.enum import {", ".join(enum_names)}
-from extra.assembly.amd.pcode import *
-''']
-
-  instructions: dict = {cls: {} for cls in OP_ENUMS}
-  for key, pc in pseudocode.items():
-    if key in defined_ops:
-      for enum_cls, enum_val in defined_ops[key]: instructions[enum_cls][enum_val] = pc
-
-  for enum_cls in OP_ENUMS:
-    cls_name = enum_cls.__name__
-    if not instructions.get(enum_cls): continue
-    fn_entries = []
-    for op, pc in instructions[enum_cls].items():
-      if any(p in pc for p in UNSUPPORTED): continue
-      try:
-        code = compile_pseudocode(pc)
-        code = _apply_pseudocode_fixes(op, code)
-        fn_name, fn_code = _generate_function(cls_name, op, pc, code)
-        lines.append(fn_code)
-        fn_entries.append((op, fn_name))
-      except Exception as e: print(f"  Warning: Failed to compile {op.name}: {e}")
-    if fn_entries:
-      lines.append(f'{cls_name}_FUNCTIONS = {{')
-      for op, fn_name in fn_entries: lines.append(f"  {cls_name}.{op.name}: {fn_name},")
-      lines.append('}\n')
-
-  # Add V_WRITELANE_B32 if VOP3Op exists
-  if 'VOP3Op' in enum_names:
-    lines.append('''
-# V_WRITELANE_B32: Write scalar to specific lane's VGPR (not in PDF pseudocode)
-def _VOP3Op_V_WRITELANE_B32(s0, s1, s2, d0, scc, vcc, lane, exec_mask, literal, VGPR, _vars, src0_idx=0, vdst_idx=0):
-  wr_lane = s1 & 0x1f
-  return {'d0': d0, 'scc': scc, 'vgpr_write': (wr_lane, vdst_idx, s0 & 0xffffffff)}
-VOP3Op_FUNCTIONS[VOP3Op.V_WRITELANE_B32] = _VOP3Op_V_WRITELANE_B32
-''')
-
-  lines.append('COMPILED_FUNCTIONS = {')
-  for enum_cls in OP_ENUMS:
-    if instructions.get(enum_cls): lines.append(f'  {enum_cls.__name__}: {enum_cls.__name__}_FUNCTIONS,')
-  lines.append('}\n\ndef get_compiled_functions(): return COMPILED_FUNCTIONS')
-  return '\n'.join(lines)
-
-def _apply_pseudocode_fixes(op, code: str) -> str:
-  """Apply known fixes for PDF pseudocode bugs."""
-  if op.name == 'V_DIV_FMAS_F32':
-    code = code.replace('D0.f32 = 2.0 ** 32 * fma(S0.f32, S1.f32, S2.f32)',
-                        'D0.f32 = (2.0 ** 64 if exponent(S2.f32) > 127 else 2.0 ** -64) * fma(S0.f32, S1.f32, S2.f32)')
-  if op.name == 'V_DIV_FMAS_F64':
-    code = code.replace('D0.f64 = 2.0 ** 64 * fma(S0.f64, S1.f64, S2.f64)',
-                        'D0.f64 = (2.0 ** 128 if exponent(S2.f64) > 1023 else 2.0 ** -128) * fma(S0.f64, S1.f64, S2.f64)')
-  if op.name == 'V_DIV_SCALE_F32':
-    code = code.replace('D0.f32 = float("nan")', 'VCC = Reg(0x1); D0.f32 = float("nan")')
-    code = code.replace('elif S1.f32 == DENORM.f32:\n  D0.f32 = ldexp(S0.f32, 64)', 'elif False:\n  pass')
-    code += '\nif S1.f32 == DENORM.f32:\n  D0.f32 = float("nan")'
-    code = code.replace('elif exponent(S2.f32) <= 23:\n  D0.f32 = ldexp(S0.f32, 64)', 'elif exponent(S2.f32) <= 23:\n  VCC = Reg(0x1); D0.f32 = ldexp(S0.f32, 64)')
-    code = code.replace('elif S2.f32 / S1.f32 == DENORM.f32:\n  VCC = Reg(0x1)\n  if S0.f32 == S2.f32:\n    D0.f32 = ldexp(S0.f32, 64)', 'elif S2.f32 / S1.f32 == DENORM.f32:\n  VCC = Reg(0x1)')
-  if op.name == 'V_DIV_SCALE_F64':
-    code = code.replace('D0.f64 = float("nan")', 'VCC = Reg(0x1); D0.f64 = float("nan")')
-    code = code.replace('elif S1.f64 == DENORM.f64:\n  D0.f64 = ldexp(S0.f64, 128)', 'elif False:\n  pass')
-    code += '\nif S1.f64 == DENORM.f64:\n  D0.f64 = float("nan")'
-    code = code.replace('elif exponent(S2.f64) <= 52:\n  D0.f64 = ldexp(S0.f64, 128)', 'elif exponent(S2.f64) <= 52:\n  VCC = Reg(0x1); D0.f64 = ldexp(S0.f64, 128)')
-    code = code.replace('elif S2.f64 / S1.f64 == DENORM.f64:\n  VCC = Reg(0x1)\n  if S0.f64 == S2.f64:\n    D0.f64 = ldexp(S0.f64, 128)', 'elif S2.f64 / S1.f64 == DENORM.f64:\n  VCC = Reg(0x1)')
-  if op.name == 'V_DIV_FIXUP_F32':
-    code = code.replace('D0.f32 = ((-abs(S0.f32)) if (sign_out) else (abs(S0.f32)))',
-                        'D0.f32 = ((-OVERFLOW_F32) if (sign_out) else (OVERFLOW_F32)) if isNAN(S0.f32) else ((-abs(S0.f32)) if (sign_out) else (abs(S0.f32)))')
-  if op.name == 'V_DIV_FIXUP_F64':
-    code = code.replace('D0.f64 = ((-abs(S0.f64)) if (sign_out) else (abs(S0.f64)))',
-                        'D0.f64 = ((-OVERFLOW_F64) if (sign_out) else (OVERFLOW_F64)) if isNAN(S0.f64) else ((-abs(S0.f64)) if (sign_out) else (abs(S0.f64)))')
-  if op.name == 'V_TRIG_PREOP_F64':
-    code = code.replace('result = F((TWO_OVER_PI_1201[1200 : 0] << shift.u32) & 0x1fffffffffffff)',
-                        'result = float(((TWO_OVER_PI_1201[1200 : 0] << int(shift)) >> (1201 - 53)) & 0x1fffffffffffff)')
-  return code
-
-def _generate_function(cls_name: str, op, pc: str, code: str) -> tuple[str, str]:
-  """Generate a single compiled pseudocode function."""
-  has_d1 = '{ D1' in pc
-  is_cmpx = (cls_name in ('VOPCOp', 'VOP3Op')) and 'EXEC.u64[laneId]' in pc
-  is_div_scale = 'DIV_SCALE' in op.name
-  has_sdst = cls_name == 'VOP3SDOp' and ('VCC.u64[laneId]' in pc or is_div_scale)
-  has_opsel = 'OPSEL' in pc  # FMA_MIX and similar instructions need OPSEL/OPSEL_HI
-  combined = code + pc
-
-  fn_name = f"_{cls_name}_{op.name}"
-  # Function accepts Reg objects directly (uppercase names), laneId is passed directly as int
-  params = "S0, S1, S2, D0, SCC, VCC, laneId, EXEC, literal, VGPR, src0_idx=0, vdst_idx=0, PC=None"
-  if has_opsel: params += ", OPSEL=0, OPSEL_HI=0"
-  lines = [f"def {fn_name}({params}):"]
-
-  # Registers that need special handling (not passed directly)
-  # Only init if used but not first assigned as `name = Reg(...)` in the compiled code
-  def needs_init(name): return name in combined and not re.search(rf'^\s*{name}\s*=\s*Reg\(', code, re.MULTILINE)
-  special_regs = [('D1', 'Reg(0)'), ('SIMM16', 'Reg(literal)'), ('SIMM32', 'Reg(literal)'),
-                  ('SRC0', 'Reg(src0_idx)'), ('VDST', 'Reg(vdst_idx)')]
-  if needs_init('tmp'): special_regs.insert(0, ('tmp', 'Reg(0)'))
-  if needs_init('saveexec'): special_regs.insert(0, ('saveexec', 'Reg(EXEC._val)'))
-  used = {name for name, _ in special_regs if name in combined}
-
-  # Detect which registers are modified (not just read) - look for assignments
-  modifies_d0 = is_div_scale or bool(re.search(r'\bD0\b[.\[]', combined))
-  modifies_exec = is_cmpx or bool(re.search(r'EXEC\.(u32|u64|b32|b64)\s*=', combined))
-  modifies_vcc = has_sdst or bool(re.search(r'VCC\.(u32|u64|b32|b64)\s*=|VCC\.u64\[laneId\]\s*=', combined))
-  modifies_scc = bool(re.search(r'\bSCC\s*=', combined))
-  modifies_pc = bool(re.search(r'\bPC\s*=', combined))
-
-  # Build init code for special registers
-  init_lines = []
-  if is_div_scale: init_lines.append("  D0 = Reg(S0._val)")
-  for name, init in special_regs:
-    if name in used: init_lines.append(f"  {name} = {init}")
-  if 'EXEC_LO' in code: init_lines.append("  EXEC_LO = SliceProxy(EXEC, 31, 0)")
-  if 'EXEC_HI' in code: init_lines.append("  EXEC_HI = SliceProxy(EXEC, 63, 32)")
-  if 'VCCZ' in code and not re.search(r'^\s*VCCZ\s*=', code, re.MULTILINE): init_lines.append("  VCCZ = Reg(1 if VCC._val == 0 else 0)")
-  if 'EXECZ' in code and not re.search(r'^\s*EXECZ\s*=', code, re.MULTILINE): init_lines.append("  EXECZ = Reg(1 if EXEC._val == 0 else 0)")
-  code_lines = [line for line in code.split('\n') if line.strip()]
-  if init_lines:
-    lines.extend(init_lines)
-    if code_lines: lines.append("  # --- compiled pseudocode ---")
-  for line in code_lines:
-    lines.append(f"  {line}")
-
-  # Build result dict - only include registers that are modified
-  result_items = []
-  if modifies_d0: result_items.append("'D0': D0")
-  if modifies_scc: result_items.append("'SCC': SCC")
-  if modifies_vcc: result_items.append("'VCC': VCC")
-  if modifies_exec: result_items.append("'EXEC': EXEC")
-  if has_d1: result_items.append("'D1': D1")
-  if modifies_pc: result_items.append("'PC': PC")
-  lines.append(f"  return {{{', '.join(result_items)}}}\n")
-  return fn_name, '\n'.join(lines)
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# MAIN GENERATION
-# ═══════════════════════════════════════════════════════════════════════════════
-
-def generate_arch(arch: str) -> dict:
-  """Generate enum.py, ins.py and gen_pcode.py for a single architecture."""
-  urls = PDF_URLS[arch]
-  if isinstance(urls, str): urls = [urls]
-
-  print(f"\n{'='*60}\nGenerating {arch}...")
-  print(f"Parsing {len(urls)} PDF(s)...")
-  results = [_parse_single_pdf(url) for url in urls]
-  merged = _merge_results(results) if len(results) > 1 else results[0]
-  doc_name = "+".join(merged["doc_names"]) if len(results) > 1 else merged["doc_name"]
-
-  base_path = Path(f"extra/assembly/amd/autogen/{arch}")
-  base_path.mkdir(parents=True, exist_ok=True)
-  (base_path / "__init__.py").touch()
-
-  # Write enum.py (enums only, no dsl.py dependency)
-  enum_path = base_path / "enum.py"
-  enum_content = _generate_enum_py(merged["enums"], merged["src_enum"], doc_name)
-  enum_path.write_text(enum_content)
-  print(f"Generated {enum_path}: SrcEnum ({len(merged['src_enum'])}) + {len(merged['enums'])} enums")
-
-  # Write ins.py (instruction formats and helpers, imports dsl.py and enum.py)
-  ins_path = base_path / "ins.py"
-  ins_content = _generate_ins_py(merged["formats"], merged["enums"], merged["src_enum"], doc_name).replace("{arch}", arch)
-  ins_path.write_text(ins_content)
-  print(f"Generated {ins_path}: {len(merged['formats'])} formats")
-
-  # Write gen_pcode.py (needs enum.py to exist first for imports)
-  pcode_path = base_path / "gen_pcode.py"
-  pcode_content = _generate_gen_pcode_py(merged["enums"], merged["pseudocode"], arch)
-  pcode_path.write_text(pcode_content)
-  print(f"Generated {pcode_path}: {len(merged['pseudocode'])} instructions")
-
-  return merged
-
-def _generate_arch_wrapper(arch: str):
-  """Wrapper for multiprocessing - returns arch name for ordering."""
-  generate_arch(arch)
-  return arch
-
-def generate_all():
-  """Generate all architectures in parallel."""
-  with ProcessPoolExecutor() as executor:
-    list(executor.map(_generate_arch_wrapper, PDF_URLS.keys()))
-
-if __name__ == "__main__":
-  import argparse
-  parser = argparse.ArgumentParser(description="Generate AMD ISA autogen files from PDF documentation")
-  parser.add_argument("--arch", choices=list(PDF_URLS.keys()) + ["all"], default="rdna3")
-  args = parser.parse_args()
-  if args.arch == "all": generate_all()
-  else: generate_arch(args.arch)

extra/assembly/amd/test/external_test_usability.py

@@ -3,7 +3,7 @@
 # Currently many of these tests fail - they document desired behavior
 
 import unittest
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.dsl import Inst, RawImm, SGPR, VGPR
 
 class TestRegisterSliceSyntax(unittest.TestCase):

extra/assembly/amd/test/helpers.py

@@ -22,45 +22,3 @@ def get_llvm_objdump():
   for p in ['llvm-objdump', 'llvm-objdump-21', 'llvm-objdump-20']:
     if shutil.which(p): return p
   raise FileNotFoundError("llvm-objdump not found")
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# EXECUTION CONTEXT (for testing compiled pseudocode)
-# ═══════════════════════════════════════════════════════════════════════════════
-
-class ExecContext:
-  """Context for running compiled pseudocode in tests."""
-  def __init__(self, s0=0, s1=0, s2=0, d0=0, scc=0, vcc=0, lane=0, exec_mask=0xffffffff, literal=0, vgprs=None, src0_idx=0, vdst_idx=0):
-    from extra.assembly.amd.pcode import Reg, MASK32, MASK64, SliceProxy
-    self._Reg, self._MASK64, self._SliceProxy = Reg, MASK64, SliceProxy
-    self.S0, self.S1, self.S2 = Reg(s0), Reg(s1), Reg(s2)
-    self.D0, self.D1 = Reg(d0), Reg(0)
-    self.SCC, self.VCC, self.EXEC = Reg(scc), Reg(vcc), Reg(exec_mask)
-    self.tmp, self.saveexec = Reg(0), Reg(exec_mask)
-    self.lane, self.laneId, self.literal = lane, lane, literal
-    self.SIMM16, self.SIMM32 = Reg(literal), Reg(literal)
-    self.VGPR = vgprs if vgprs is not None else {}
-    self.SRC0, self.VDST = Reg(src0_idx), Reg(vdst_idx)
-
-  def run(self, code: str):
-    """Execute compiled code."""
-    import extra.assembly.amd.pcode as pcode
-    ns = {k: getattr(pcode, k) for k in dir(pcode) if not k.startswith('_')}
-    # Also include underscore-prefixed helpers that compiled pseudocode uses
-    for k in ['_pack', '_pack32']:
-      if hasattr(pcode, k): ns[k] = getattr(pcode, k)
-    ns.update({
-      'S0': self.S0, 'S1': self.S1, 'S2': self.S2, 'D0': self.D0, 'D1': self.D1,
-      'SCC': self.SCC, 'VCC': self.VCC, 'EXEC': self.EXEC,
-      'EXEC_LO': self._SliceProxy(self.EXEC, 31, 0), 'EXEC_HI': self._SliceProxy(self.EXEC, 63, 32),
-      'tmp': self.tmp, 'saveexec': self.saveexec,
-      'lane': self.lane, 'laneId': self.laneId, 'literal': self.literal,
-      'SIMM16': self.SIMM16, 'SIMM32': self.SIMM32, 'VGPR': self.VGPR, 'SRC0': self.SRC0, 'VDST': self.VDST,
-    })
-    exec(code, ns)
-    def _sync(ctx_reg, ns_val):
-      if isinstance(ns_val, self._Reg): ctx_reg._val = ns_val._val
-      else: ctx_reg._val = int(ns_val) & self._MASK64
-    for name in ('SCC', 'VCC', 'EXEC', 'D0', 'D1', 'tmp', 'saveexec'):
-      if ns.get(name) is not getattr(self, name): _sync(getattr(self, name), ns[name])
-
-  def result(self) -> dict: return {"d0": self.D0._val, "scc": self.SCC._val & 1}

extra/assembly/amd/test/test_compare_emulators.py

@@ -107,16 +107,16 @@ class PythonEmulator:
     return step_wave(self.program, self.state, self.lds, self.n_lanes)
   def set_sgpr(self, idx: int, val: int):
     assert self.state is not None
-    self.state.sgpr[idx] = val & 0xffffffff
+    self.state.sgpr[idx]._val = val & 0xffffffff
   def set_vgpr(self, lane: int, idx: int, val: int):
     assert self.state is not None
-    self.state.vgpr[lane][idx] = val & 0xffffffff
+    self.state.vgpr[lane][idx]._val = val & 0xffffffff
 
   def get_snapshot(self) -> StateSnapshot:
     assert self.state is not None
     return StateSnapshot(pc=self.state.pc, scc=self.state.scc, vcc=self.state.vcc & 0xffffffff,
-                         exec_mask=self.state.exec_mask & 0xffffffff, sgpr=list(self.state.sgpr),
-                         vgpr=[list(self.state.vgpr[i]) for i in range(WAVE_SIZE)])
+                         exec_mask=self.state.exec_mask & 0xffffffff, sgpr=[r._val for r in self.state.sgpr],
+                         vgpr=[[r._val for r in self.state.vgpr[i]] for i in range(WAVE_SIZE)])
 
 def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: tuple[int, int, int],
                       program, max_steps: int, debug: bool, trace_len: int, kernel_idx: int = 0,
@@ -191,9 +191,6 @@ def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: t
             python_result = python.step()
 
             if rust_result != python_result:
-              # Rust returns 1 for unsupported instructions - skip test
-              if rust_result == 1 and python_result == 0:
-                raise unittest.SkipTest(f"Rust emulator doesn't support instruction: {inst_str}")
               trace_str = "\n".join(f"    step {s}: PC={pc:3d} {d}" for s, pc, d, _, _ in trace)
               return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: different return codes: rust={rust_result}, python={python_result}, inst={inst_str}\n  Recent instructions:\n{trace_str}", total_steps
 
@@ -364,7 +361,6 @@ class TestTinygradKernels(unittest.TestCase):
 
   # Matmul
   def test_gemm(self): self._test_kernel(lambda T: T.empty(8, 8) @ T.empty(8, 8), max_steps=100000)
-  @unittest.skip("Rust emulator crashes on this kernel (assertion failure in thread.rs)")
   def test_gemm_fp16(self): self._test_kernel(lambda T: T.empty(16, 16).half() @ T.empty(16, 16).half(), max_steps=100000)
 
   # Complex ops

extra/assembly/amd/test/test_formats.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 """Test MUBUF, MTBUF, MIMG, EXP, DS formats against LLVM."""
 import unittest
-from extra.assembly.amd.autogen.rdna3.ins import *
-from extra.assembly.amd.dsl import encode_src, RawImm
-from extra.assembly.amd.asm import detect_format
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.dsl import encode_src
 
 class TestMUBUF(unittest.TestCase):
   """Test MUBUF (buffer) instructions."""
@@ -329,79 +328,5 @@ class TestVOP3Literal(unittest.TestCase):
     self.assertEqual(len(inst.to_bytes()), 12)  # 8 bytes + 4 byte literal
 
 
-class TestDetectFormat(unittest.TestCase):
-  """Test detect_format uses encoding from autogen classes."""
-
-  def test_detect_sopp(self):
-    self.assertEqual(detect_format(s_endpgm().to_bytes()), SOPP)
-    self.assertEqual(detect_format(s_nop(0).to_bytes()), SOPP)
-    self.assertEqual(detect_format(s_barrier().to_bytes()), SOPP)
-
-  def test_detect_sop1(self):
-    self.assertEqual(detect_format(s_mov_b32(s[0], 0).to_bytes()), SOP1)
-    self.assertEqual(detect_format(s_mov_b64(s[0:1], 0).to_bytes()), SOP1)
-
-  def test_detect_sop2(self):
-    self.assertEqual(detect_format(s_add_u32(s[0], s[1], s[2]).to_bytes()), SOP2)
-    self.assertEqual(detect_format(s_mul_i32(s[0], s[1], s[2]).to_bytes()), SOP2)
-
-  def test_detect_sopc(self):
-    self.assertEqual(detect_format(s_cmp_eq_i32(s[0], s[1]).to_bytes()), SOPC)
-
-  def test_detect_sopk(self):
-    self.assertEqual(detect_format(s_movk_i32(s[0], 0x1234).to_bytes()), SOPK)
-
-  def test_detect_vop1(self):
-    self.assertEqual(detect_format(v_mov_b32_e32(v[0], 0).to_bytes()), VOP1)
-    self.assertEqual(detect_format(v_rcp_f32_e32(v[0], v[1]).to_bytes()), VOP1)
-
-  def test_detect_vop2(self):
-    self.assertEqual(detect_format(v_add_f32_e32(v[0], v[1], v[2]).to_bytes()), VOP2)
-    self.assertEqual(detect_format(v_mul_f32_e32(v[0], v[1], v[2]).to_bytes()), VOP2)
-
-  def test_detect_vopc(self):
-    self.assertEqual(detect_format(v_cmp_eq_f32_e32(v[0], v[1]).to_bytes()), VOPC)
-    self.assertEqual(detect_format(v_cmp_lt_i32_e32(v[0], v[1]).to_bytes()), VOPC)
-
-  def test_detect_vop3(self):
-    self.assertEqual(detect_format(v_add_f32_e64(v[0], v[1], v[2]).to_bytes()), VOP3)
-    self.assertEqual(detect_format(v_fma_f32(v[0], v[1], v[2], v[3]).to_bytes()), VOP3)
-
-  def test_detect_vop3p(self):
-    self.assertEqual(detect_format(VOP3P(VOP3POp.V_PK_ADD_F16, v[0], v[1], v[2], v[3]).to_bytes()), VOP3P)
-
-  def test_detect_smem(self):
-    self.assertEqual(detect_format(s_load_b32(s[0], s[2:3], 0).to_bytes()), SMEM)
-    self.assertEqual(detect_format(s_load_b64(s[0:1], s[2:3], s[5]).to_bytes()), SMEM)
-
-  def test_detect_ds(self):
-    self.assertEqual(detect_format(ds_load_b32(v[0], v[1]).to_bytes()), DS)
-    self.assertEqual(detect_format(ds_store_b32(v[0], v[1]).to_bytes()), DS)
-
-  def test_detect_flat(self):
-    self.assertEqual(detect_format(global_load_b32(v[0], v[1:3], RawImm(124)).to_bytes()), FLAT)
-    self.assertEqual(detect_format(global_store_b32(v[0:2], v[2], RawImm(124)).to_bytes()), FLAT)
-
-  def test_detect_mubuf(self):
-    self.assertEqual(detect_format(buffer_load_b32(v[0], v[1], s[0:4], s[5]).to_bytes()), MUBUF)
-
-  def test_detect_mtbuf(self):
-    self.assertEqual(detect_format(tbuffer_load_format_x(v[0], v[1], s[0:4], s[5], format=22).to_bytes()), MTBUF)
-
-  def test_detect_mimg(self):
-    self.assertEqual(detect_format(image_load(v[0:4], v[4:6], s[0:8], dmask=0xf, dim=1).to_bytes()), MIMG)
-
-  def test_detect_exp(self):
-    self.assertEqual(detect_format(EXP(en=0xf, target=0, vsrc0=v[0], vsrc1=v[1], vsrc2=v[2], vsrc3=v[3]).to_bytes()), EXP)
-
-  def test_detect_vopd(self):
-    inst = VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=0, srcy0=0)
-    self.assertEqual(detect_format(inst.to_bytes()), VOPD)
-
-  def test_detect_vinterp(self):
-    inst = VINTERP(VINTERPOp.V_INTERP_P10_F32, vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
-    self.assertEqual(detect_format(inst.to_bytes()), VINTERP)
-
-
 if __name__ == "__main__":
   unittest.main()

extra/assembly/amd/test/test_handwritten.py

@@ -2,7 +2,7 @@
 # the Inst constructor should be looking at the types of the fields to correctly set the value
 
 import unittest, struct
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.dsl import Inst
 from extra.assembly.amd.asm import asm
 from extra.assembly.amd.test.test_roundtrip import compile_asm

extra/assembly/amd/test/test_integration.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Integration test: round-trip RDNA3 assembly through AMD toolchain."""
 import unittest, re, io, sys, subprocess
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.asm import waitcnt, asm
 from extra.assembly.amd.test.helpers import get_llvm_mc
 

extra/assembly/amd/test/test_llvm.py

@@ -2,7 +2,7 @@
 """Test RDNA3 assembler/disassembler against LLVM test vectors."""
 import unittest, re, subprocess
 from tinygrad.helpers import fetch
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.asm import asm
 from extra.assembly.amd.test.helpers import get_llvm_mc
 
@@ -65,18 +65,12 @@ def parse_llvm_tests(text: str) -> list[tuple[str, bytes]]:
     if not asm_text: continue
     for j in range(i, min(i + 3, len(lines))):
       # Match GFX11, W32, or W64 encodings (all valid for gfx11)
-      # Format 1: "// GFX11: v_foo ... ; encoding: [0x01,0x02,...]"
-      # Format 2: "// GFX11: [0x01,0x02,...]" (used by DS, older files)
       if m := re.search(r'(?:GFX11|W32|W64)[^:]*:.*?encoding:\s*\[(.*?)\]', lines[j]):
         hex_bytes = m.group(1).replace('0x', '').replace(',', '').replace(' ', '')
-      elif m := re.search(r'(?:GFX11|W32|W64)[^:]*:\s*\[(0x[0-9a-fA-F,x\s]+)\]', lines[j]):
-        hex_bytes = m.group(1).replace('0x', '').replace(',', '').replace(' ', '')
-      else:
-        continue
-      if hex_bytes:
-        try: tests.append((asm_text, bytes.fromhex(hex_bytes)))
-        except ValueError: pass
-      break
+        if hex_bytes:
+          try: tests.append((asm_text, bytes.fromhex(hex_bytes)))
+          except ValueError: pass
+        break
   return tests
 
 def try_assemble(text: str):

extra/assembly/amd/test/test_pcode.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 """Tests for the RDNA3 pseudocode DSL."""
 import unittest
-from extra.assembly.amd.pcode import (Reg, TypedView, SliceProxy, MASK32, MASK64,
+from extra.assembly.amd.pcode import (Reg, TypedView, SliceProxy, ExecContext, compile_pseudocode, _expr, MASK32, MASK64,
                                        _f32, _i32, _f16, _i16, f32_to_f16, _isnan, _bf16, _ibf16, bf16_to_f32, f32_to_bf16,
                                        BYTE_PERMUTE, v_sad_u8, v_msad_u8)
-from extra.assembly.amd.pdf import compile_pseudocode, _expr
-from extra.assembly.amd.test.helpers import ExecContext
 from extra.assembly.amd.autogen.rdna3.gen_pcode import _VOP3SDOp_V_DIV_SCALE_F32, _VOPCOp_V_CMP_CLASS_F32
 
 class TestReg(unittest.TestCase):
@@ -210,8 +208,6 @@ D0.u32 = tmp.u32""")
 for i in 0 : 31 do
   if S0.u32[i] == 1 then
     tmp = i
-  endif
-endfor
 D0.i32 = tmp""")
     ctx = ExecContext(s0=0b1000)  # Bit 3 is set
     ctx.run(code)
@@ -229,18 +225,17 @@ class TestPseudocodeRegressions(unittest.TestCase):
   """Regression tests for pseudocode instruction emulation bugs."""
 
   def test_v_div_scale_f32_vcc_always_returned(self):
-    """V_DIV_SCALE_F32 must always return VCC, even when VCC=0 (no scaling needed).
-    Bug: when VCC._val == vcc (both 0), VCC wasn't returned, so VCC bits weren't written.
-    This caused division to produce wrong results for multiple lanes."""
+    """V_DIV_SCALE_F32 must set VCC bit for the lane when scaling is needed.
+    The new calling convention uses Reg objects and modifies VCC in place."""
     # Normal case: 1.0 / 3.0, no scaling needed, VCC should be 0
     S0 = Reg(0x3f800000)  # 1.0
     S1 = Reg(0x40400000)  # 3.0
     S2 = Reg(0x3f800000)  # 1.0 (numerator)
-    D0, SCC, VCC, EXEC = Reg(0), Reg(0), Reg(0), Reg(0xffffffff)
-    result = _VOP3SDOp_V_DIV_SCALE_F32(S0, S1, S2, D0, SCC, VCC, 0, EXEC, 0, None)
-    # Must always have VCC in result
-    self.assertIn('VCC', result, "V_DIV_SCALE_F32 must always return VCC")
-    self.assertEqual(result['VCC']._val & 1, 0, "VCC lane 0 should be 0 when no scaling needed")
+    D0 = Reg(0)
+    VCC = Reg(0)
+    _VOP3SDOp_V_DIV_SCALE_F32(S0, S1, S2, D0, Reg(0), VCC, 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    # VCC bit 0 should be 0 when no scaling needed
+    self.assertEqual(VCC._val & 1, 0, "VCC bit should be 0 when no scaling needed")
 
   def test_v_cmp_class_f32_detects_quiet_nan(self):
     """V_CMP_CLASS_F32 must correctly identify quiet NaN vs signaling NaN.
@@ -249,22 +244,22 @@ class TestPseudocodeRegressions(unittest.TestCase):
     signal_nan = 0x7f800001  # signaling NaN: exponent=255, bit22=0
     # Test quiet NaN detection (bit 1 in mask)
     s1_quiet = 0b0000000010  # bit 1 = quiet NaN
-    S0, S1, S2, D0, SCC, VCC, EXEC = Reg(quiet_nan), Reg(s1_quiet), Reg(0), Reg(0), Reg(0), Reg(0), Reg(0xffffffff)
-    result = _VOPCOp_V_CMP_CLASS_F32(S0, S1, S2, D0, SCC, VCC, 0, EXEC, 0, None)
-    self.assertEqual(result['D0']._val & 1, 1, "Should detect quiet NaN with quiet NaN mask")
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(quiet_nan), Reg(s1_quiet), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 1, "Should detect quiet NaN with quiet NaN mask")
     # Test signaling NaN detection (bit 0 in mask)
     s1_signal = 0b0000000001  # bit 0 = signaling NaN
-    S0, S1 = Reg(signal_nan), Reg(s1_signal)
-    result = _VOPCOp_V_CMP_CLASS_F32(S0, S1, S2, D0, SCC, VCC, 0, EXEC, 0, None)
-    self.assertEqual(result['D0']._val & 1, 1, "Should detect signaling NaN with signaling NaN mask")
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(signal_nan), Reg(s1_signal), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 1, "Should detect signaling NaN with signaling NaN mask")
     # Test that quiet NaN doesn't match signaling NaN mask
-    S0, S1 = Reg(quiet_nan), Reg(s1_signal)
-    result = _VOPCOp_V_CMP_CLASS_F32(S0, S1, S2, D0, SCC, VCC, 0, EXEC, 0, None)
-    self.assertEqual(result['D0']._val & 1, 0, "Quiet NaN should not match signaling NaN mask")
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(quiet_nan), Reg(s1_signal), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 0, "Quiet NaN should not match signaling NaN mask")
     # Test that signaling NaN doesn't match quiet NaN mask
-    S0, S1 = Reg(signal_nan), Reg(s1_quiet)
-    result = _VOPCOp_V_CMP_CLASS_F32(S0, S1, S2, D0, SCC, VCC, 0, EXEC, 0, None)
-    self.assertEqual(result['D0']._val & 1, 0, "Signaling NaN should not match quiet NaN mask")
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(signal_nan), Reg(s1_quiet), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 0, "Signaling NaN should not match quiet NaN mask")
 
   def test_isnan_with_typed_view(self):
     """_isnan must work with TypedView objects, not just Python floats.

extra/assembly/amd/test/test_pdf_parser.py

@@ -1,7 +1,10 @@
 #!/usr/bin/env python3
 """Test that PDF parser correctly extracts format fields."""
 import unittest, os
-from extra.assembly.amd.autogen.rdna3.ins import SOP1, SOP2, SOPK, SOPP, VOP1, VOP2, VOP3SD, VOPC, FLAT, VOPD, SOP1Op, SOP2Op, VOP1Op, VOP3Op
+from extra.assembly.amd.autogen.rdna3 import (
+  SOP1, SOP2, SOPK, SOPP, VOP1, VOP2, VOP3SD, VOPC, FLAT, VOPD,
+  SOP1Op, SOP2Op, VOP1Op, VOP3Op
+)
 
 # expected formats with key fields and whether they have ENCODING
 EXPECTED_FORMATS = {

extra/assembly/amd/test/test_rdna3_asm.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import unittest, subprocess
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.test.helpers import get_llvm_mc
 
 def llvm_assemble(asm: str) -> bytes:

extra/assembly/amd/test/test_roundtrip.py

@@ -1,12 +1,49 @@
 #!/usr/bin/env python3
 """Roundtrip tests: generate tinygrad kernels, decode instructions, re-encode, verify match."""
 import unittest, io, sys, re, subprocess, os
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.dsl import Inst
 from extra.assembly.amd.asm import asm
-from extra.assembly.amd.asm import detect_format
 from extra.assembly.amd.test.helpers import get_llvm_mc, get_llvm_objdump
 
+# Instruction format detection based on encoding bits
+def detect_format(data: bytes) -> type[Inst] | None:
+  """Detect instruction format from machine code bytes."""
+  if len(data) < 4: return None
+  word = int.from_bytes(data[:4], 'little')
+  enc_9bit = (word >> 23) & 0x1FF  # 9-bit encoding for SOP1/SOPC/SOPP
+  enc_8bit = (word >> 24) & 0xFF
+
+  # Check 9-bit encodings first (most specific)
+  if enc_9bit == 0x17D: return SOP1  # bits 31:23 = 101111101
+  if enc_9bit == 0x17E: return SOPC  # bits 31:23 = 101111110
+  if enc_9bit == 0x17F: return SOPP  # bits 31:23 = 101111111
+  # SOPK: bits 31:28 = 1011, bits 27:23 = opcode (check after SOP1/SOPC/SOPP)
+  if enc_8bit in range(0xB0, 0xC0): return SOPK
+  # SOP2: bits 31:23 in range 0x100-0x17C (0x80-0xBE in bits 31:24, but not SOPK)
+  if 0x80 <= enc_8bit <= 0x9F: return SOP2
+  # VOP1: bits 31:25 = 0111111 (0x3F)
+  if (word >> 25) == 0x3F: return VOP1
+  # VOPC: bits 31:25 = 0111110 (0x3E)
+  if (word >> 25) == 0x3E: return VOPC
+  # VOP2: bits 31:30 = 00
+  if (word >> 30) == 0: return VOP2
+
+  # Check 64-bit formats
+  if len(data) >= 8:
+    if enc_8bit in (0xD4, 0xD5, 0xD7): return VOP3
+    if enc_8bit == 0xD6: return VOP3SD
+    if enc_8bit == 0xCC: return VOP3P
+    if enc_8bit == 0xCD: return VINTERP
+    if enc_8bit in (0xC8, 0xC9): return VOPD
+    if enc_8bit == 0xF4: return SMEM
+    if enc_8bit == 0xD8: return DS
+    if enc_8bit in (0xDC, 0xDD, 0xDE, 0xDF): return FLAT
+    if enc_8bit in (0xE0, 0xE1, 0xE2, 0xE3): return MUBUF
+    if enc_8bit in (0xE8, 0xE9, 0xEA, 0xEB): return MTBUF
+
+  return None
+
 def disassemble_lib(lib: bytes, compiler) -> list[tuple[str, bytes]]:
   """Disassemble ELF binary and return list of (instruction_text, machine_code_bytes)."""
   old_stdout = sys.stdout

extra/gemm/asm/template.s

@@ -1,83 +0,0 @@
-.text
-.section	.text.
-.global	gemm
-.p2align	8
-.type	gemm,@function
-
-gemm:
-INSTRUCTIONS
-
-.section .rodata,"a",@progbits
-.p2align 6, 0x0
-.amdhsa_kernel gemm
-  # basic memory requirements
-  .amdhsa_group_segment_fixed_size 133120
-  .amdhsa_private_segment_fixed_size 0
-  .amdhsa_kernarg_size 32
-  # register usage (RSRC1)
-  .amdhsa_next_free_vgpr 504
-  .amdhsa_next_free_sgpr 96
-  # workgroup / workitem IDs (RSRC2)
-  .amdhsa_system_sgpr_workgroup_id_x 1
-  .amdhsa_system_sgpr_workgroup_id_y 1
-  .amdhsa_system_sgpr_workgroup_id_z 1
-  # user SGPRs, we only specify the kernel args ptr in s[0:1]
-  .amdhsa_user_sgpr_kernarg_segment_ptr 1
-  .amdhsa_user_sgpr_count 2
-  .amdhsa_user_sgpr_kernarg_preload_length 0
-  .amdhsa_user_sgpr_kernarg_preload_offset 0
-  # gfx90a / gfx940 specifics (RSRC3)
-  .amdhsa_accum_offset 248
-  .amdhsa_uses_dynamic_stack 0
-  .amdhsa_tg_split 0
-.end_amdhsa_kernel
-
-.amdgpu_metadata
----
-amdhsa.kernels:
-  - .name: gemm
-    .symbol: gemm.kd
-    .args:
-      - .name: C
-        .address_space: global
-        .offset: 0
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: B
-        .address_space: global
-        .offset: 8
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: A
-        .address_space: global
-        .offset: 16
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: sz
-        .offset: 24
-        .size: 4
-        .value_kind: by_value
-        .value_type: u32
-      - .name: num_wg
-        .offset: 28
-        .size: 4
-        .value_kind: by_value
-        .value_type: u32
-    .group_segment_fixed_size: 133120
-    .private_segment_fixed_size: 0
-    .kernarg_segment_align: 8
-    .kernarg_segment_size: 32
-    .max_flat_workgroup_size: 256
-    .sgpr_count: 88
-    .sgpr_spill_count: 0
-    .vgpr_count: 248
-    .vgpr_spill_count: 0
-    .wavefront_size: 64
-amdhsa.version:
-  - 1
-  - 0
-...
-.end_amdgpu_metadata

extra/gemm/asm/test.py

@@ -1,7 +1,7 @@
 # Run assembly on the AMD runtime and check correctness
 # VIZ=2 to profile
 import pathlib
-from tinygrad import Tensor, Device, dtypes, Context
+from tinygrad import Tensor, Device, dtypes
 from tinygrad.engine.realize import ExecItem, CompiledRunner
 from tinygrad.renderer import ProgramSpec
 from tinygrad.uop.ops import track_rewrites, UOp
@@ -48,17 +48,16 @@ ast = sched[-1].ast
 # assembly gemm
 @track_rewrites(name=lambda ret: TracingKey(ret.name, (ret.function_name,), ret))
 def get_asm_prg() -> ProgramSpec:
-  src = (pathlib.Path(__file__).parent/"template.s").read_text().replace("INSTRUCTIONS", fp.read_text())
+  src = fp.read_text()
   lib = Device[Device.DEFAULT].compiler.compile(src)
   return ProgramSpec("gemm", src, Device.DEFAULT, ast, lib=lib, global_size=[NUM_WG, 1, 1], local_size=[THREADS_PER_WG, 1, 1],
                      globals=[0, 1, 2], vars=[UOp.variable("SZ", 256, 8192), UOp.variable("NUM_WG", 1, 1024)])
 eis.append(ExecItem(ast, [C_asm.uop.buffer, from_torch(B).uop.buffer, from_torch(A).uop.buffer], fixedvars={"SZ":N, "NUM_WG":NUM_WG},
                     prg=CompiledRunner(get_asm_prg())))
 
-with Context(DEBUG=2):
-  for ei in eis:
-    et = ei.run(wait=True)
-    print(f"{(N*N*N*2 / et)*1e-12:.2f} REAL TFLOPS")
+for ei in eis:
+  et = ei.run(wait=True)
+  print(f"{(N*N*N*2 / et)*1e-12:.2f} REAL TFLOPS")
 
 # ** correctness
 

extra/mmapeak/mmapeak.py

@@ -4,8 +4,7 @@ import os, pathlib
 os.environ["AMD_AQL"] = "1"
 
 from tinygrad.device import Device
-from tinygrad.runtime.support.compiler_amd import HIPCompiler
-from tinygrad.runtime.ops_amd import AMDProgram
+from tinygrad.runtime.ops_amd import AMDProgram, HIPCompiler
 
 NUM_WORKGROUPS = 96
 WAVE_SIZE = 32

extra/remu/test/hwtest.py

@@ -11,7 +11,7 @@ from tinygrad.runtime.support.compiler_amd import amdgpu_disassemble
 from tinygrad.renderer import ProgramSpec
 from tinygrad.engine.realize import CompiledRunner
 
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 from extra.assembly.amd.asm import waitcnt
 from test.testextra.test_cfg_viz import template
 

pyproject.toml

@@ -49,7 +49,7 @@ arm = ["unicorn"]
 triton = ["triton-nightly>=2.1.0.dev20231014192330"]
 linting = [
   "pylint",
-  "mypy==1.19.1",
+  "mypy==1.18.1",
   "typing-extensions",
   "pre-commit",
   "ruff",
@@ -61,7 +61,7 @@ linting = [
 # ]
 testing_minimal = [
   "numpy",
-  "torch==2.9.1",
+  "torch==2.9.0",
   "pytest",
   "pytest-xdist",
   "pytest-timeout",
@@ -188,6 +188,7 @@ select = [
   "E72",
   "E112",   # no-indented-block
   "E113",   # unexpected-indentation
+  # "E124",
   "E203",   # whitespace-before-punctuation
   "E272",   # multiple-spaces-before-keyword
   "E275",   # missing-whitespace-after-keyword

test/device/test_qcom.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+import unittest
+from tinygrad.device import Device, BufferSpec
+from tinygrad.dtype import dtypes
+
+@unittest.skipUnless(Device.DEFAULT == "QCOM", "QCOM device required to run")
+class TestQcom(unittest.TestCase):
+  def test_image_pitch(self):
+    dev = Device["QCOM"]
+
+    def __validate(imgdt, expected_pitch):
+      img = dev.allocator.alloc(imgdt.shape[0] * imgdt.shape[1] * 16, options:=BufferSpec(image=imgdt))
+      pitch = img.texture_info.pitch
+      assert pitch == expected_pitch, f"Failed pitch for image: {imgdt}. Got 0x{pitch:X}, expected 0x{expected_pitch:X}"
+      dev.allocator.free(img, imgdt.shape[0] * imgdt.shape[1] * 16, options)
+
+    # Match opencl pitches for perf
+    __validate(dtypes.imageh((1, 201)), 0x680)
+    __validate(dtypes.imageh((16, 216)), 0x700)
+    __validate(dtypes.imageh((16, 9)), 0x80)
+    __validate(dtypes.imageh((48, 64)), 0x200)
+    __validate(dtypes.imageh((32, 128)), 0x400)
+    __validate(dtypes.imageh((96, 128)), 0x400)
+    __validate(dtypes.imageh((64, 256)), 0x840)
+    __validate(dtypes.imageh((64, 9)), 0x80)
+    __validate(dtypes.imageh((192, 256)), 0x840)
+    __validate(dtypes.imageh((64, 768)), 0x1840)
+    __validate(dtypes.imageh((256, 49)), 0x1C0)
+    __validate(dtypes.imageh((128, 9)), 0x80)
+    __validate(dtypes.imageh((16, 1024)), 0x2080)
+    __validate(dtypes.imageh((64, 512)), 0x1040)
+    __validate(dtypes.imageh((16, 512)), 0x1080)
+    __validate(dtypes.imageh((132, 64)), 0x200)
+    __validate(dtypes.imageh((4, 512)), 0x1200)
+    __validate(dtypes.imageh((8, 512)), 0x1100)
+    __validate(dtypes.imageh((128, 128)), 0x400)
+    __validate(dtypes.imageh((32, 512)), 0x1040)
+    __validate(dtypes.imageh((26, 64)), 0x200)
+    __validate(dtypes.imageh((32, 516)), 0x1040)
+    __validate(dtypes.imageh((32, 1024)), 0x2040)
+    __validate(dtypes.imageh((16, 2048)), 0x4080)
+    __validate(dtypes.imageh((8, 2048)), 0x4100)
+    __validate(dtypes.imageh((4, 4096)), 0x8200)
+
+    __validate(dtypes.imagef((16, 49)), 0x380)
+    __validate(dtypes.imagef((16, 1024)), 0x4080)
+    __validate(dtypes.imagef((256, 64)), 0x400)
+    __validate(dtypes.imagef((64, 512)), 0x2040)
+    __validate(dtypes.imagef((16, 512)), 0x2080)
+    __validate(dtypes.imagef((132, 64)), 0x400)
+    __validate(dtypes.imagef((4, 512)), 0x2200)
+    __validate(dtypes.imagef((4, 16)), 0x200)
+    __validate(dtypes.imagef((2, 16)), 0x400)
+    __validate(dtypes.imagef((8, 512)), 0x2100)
+    __validate(dtypes.imagef((12, 64)), 0x400)
+    __validate(dtypes.imagef((3, 32)), 0x400)
+    __validate(dtypes.imagef((128, 128)), 0x840)
+    __validate(dtypes.imagef((32, 512)), 0x2040)
+    __validate(dtypes.imagef((8, 3072)), 0xC100)
+    __validate(dtypes.imagef((4, 2048)), 0x8200)
+    __validate(dtypes.imagef((4, 1024)), 0x4200)
+    __validate(dtypes.imagef((4, 4096)), 0x10200)
+    __validate(dtypes.imagef((10, 384)), 0x1900)
+    __validate(dtypes.imagef((24, 64)), 0x400)
+    __validate(dtypes.imagef((128, 12)), 0xC0)
+    __validate(dtypes.imagef((10, 24)), 0x200)
+    __validate(dtypes.imagef((1, 129)), 0x840)
+    __validate(dtypes.imagef((1, 32)), 0x200)
+    __validate(dtypes.imagef((1, 64)), 0x400)
+    __validate(dtypes.imagef((1, 1239)), 0x4D80)
+    __validate(dtypes.imagef((1, 1)), 0x40)
+
+if __name__ == "__main__":
+  unittest.main()

test/external/external_benchmark_multitensor_allreduce.py

@@ -1,7 +1,7 @@
 from tinygrad import Tensor, Device, GlobalCounters, TinyJit, dtypes
-from tinygrad.helpers import getenv, Context, DEBUG
+from tinygrad.helpers import getenv, Context, RING, DEBUG
 
-def test(devs: list[str], N: int, iters:int = 10, name:str = "allreduce"):
+def test(devs: list[str], N: int, iters:int = 10):
   @TinyJit
   def f(t: Tensor) -> Tensor: t.sum(0).realize()
 
@@ -17,33 +17,39 @@ def test(devs: list[str], N: int, iters:int = 10, name:str = "allreduce"):
     i_secs = GlobalCounters.time_sum_s
     i_gflops = GlobalCounters.global_ops/i_secs/10**9
     i_gbs = (N*4)/i_secs/10**9
-    print(f"{name} iter {i+1}/{iters}: {i_secs:.6f} sec {i_gflops:.2f} GFLOP/s {i_gbs:.2f} GB/s")
+    print(f"{'ring_allreduce' if RING >= 2 else 'naive_allreduce'} iter {i+1}/{iters}: {i_secs:.6f} sec {i_gflops:.2f} GFLOP/s {i_gbs:.2f} GB/s")
     secs += i_secs
     gflops += i_gflops
     gbs += i_gbs
 
   return (gflops/iters, gbs/iters, secs/iters)
 
-def run(sz, n_gpus=6, iters=10, ring=0, all2all=0):
+def run(sz, n_gpus=6, iters=10, use_ring=False):
   devs = tuple([f"{Device.DEFAULT}:{x}" for x in range(n_gpus)])
   N = sz // dtypes.float32.itemsize
-  name = "all2all" if all2all else ("ring" if ring else "naive")
-  with Context(RING=(2 if ring else 0), ALL2ALL=(2 if all2all else 0), JIT_BATCH_SIZE=0, DEBUG=max(DEBUG.value, 2)):
-    return test(devs, N, iters=iters, name=name)
+  with Context(RING=(2 if use_ring else 0), DEBUG=max(DEBUG.value, 2)): return test(devs, N, iters=iters)
 
 def main():
+  ONLY_RING = getenv("ONLY_RING", 0)
   n_gpus = getenv("GPUS", 6)
   iters = getenv("ITERS", 10)
-  sz = getenv("SZ", 1000) * 10**6 # size of data on each gpu
-  print(f"Using {sz/10**9:.2f} GB of numbers on each of {n_gpus} GPUs, {n_gpus*sz/10**9:.2f} GB total.")
 
-  results = {}
-  for name, kwargs in [("naive", {}), ("ring", {"ring": 2}), ("all2all", {"all2all": 2})]:
-    results[name] = run(sz, n_gpus=n_gpus, iters=iters, **kwargs)
-
-  print("\n=== RESULTS ===")
-  for name, (gflops, gbs, secs) in results.items():
-    print(f"{name.upper()}:\n  {secs:.6f} seconds/iter\n  {gflops:.2f} GFLOP/s\n  {gbs:.2f} GB/s")
+  if getenv("BENCHMARK_SPLIT"):
+    l, r = 0, 512
+    while r - l > 1:
+      m = (l + r) // 2
+      (ring_gflops, ring_gbs, ring_secs) = run(m * 1024 * 4, n_gpus=n_gpus, iters=100, use_ring=True)
+      (naive_gflops, naive_gbs, naive_secs) = run(m * 1024 * 4, n_gpus=n_gpus, iters=100, use_ring=False)
+      if ring_secs > naive_secs: l = m
+      else: r = m
+    print("Better split", r * 1024, "elements")
+  else:
+    sz = getenv("SZ", 1000) * 10**6 # size of data on each gpu
+    print(f"Using {sz/10**9:.2f} GB of numbers on each of {n_gpus} GPUs, {n_gpus*sz/10**9:.2f} GB total.")
+    (ring_gflops, ring_gbs, ring_secs) = run(sz, use_ring=True, n_gpus=n_gpus, iters=iters)
+    if not ONLY_RING: (naive_gflops, naive_gbs, naive_secs) = run(sz, use_ring=False, n_gpus=n_gpus, iters=iters)
+    print(f"Ring:\n  {ring_secs:.6f} seconds/iter\n  {ring_gflops:.2f} GFLOP/s\n  {ring_gbs:.2f} GB/s")
+    if not ONLY_RING: print(f"Naive:\n  {naive_secs:.6f} seconds/iter\n  {naive_gflops:.2f} GFLOP/s\n  {naive_gbs:.2f} GB/s")
 
 if __name__ == "__main__":
   main()

test/mockgpu/amd/amdgpu.py

@@ -187,7 +187,7 @@ class PM4Executor(AMDQueue):
       if st <= prg_addr < st+sz: prg_sz = sz - (prg_addr - st)
 
     assert prg_sz > 0, "Invalid prg ptr (not found in mapped ranges)"
-    # Pass valid memory ranges and rsrc2 to Python emulator for bounds checking and SGPR/VGPR layout
+    # Pass valid memory ranges and rsrc2 to Python emulator for bounds checking and SGPR layout
     if hasattr(remu, 'valid_mem_ranges'): remu.valid_mem_ranges = self.gpu.mapped_ranges
     if hasattr(remu, 'rsrc2'): remu.rsrc2 = rsrc2
     err = remu.run_asm(prg_addr, prg_sz, *gl, *lc, args_addr)

test/test_dtype_alu.py

@@ -1,7 +1,7 @@
 import unittest, operator, math
 from tinygrad import Tensor, dtypes, Device
 from tinygrad.dtype import DType, truncate
-from tinygrad.helpers import CI, getenv
+from tinygrad.helpers import CI, getenv, CPU_LLVM
 from tinygrad.tensor import _to_np_dtype
 from tinygrad.device import is_dtype_supported
 from tinygrad.runtime.ops_python import from_storage_scalar
@@ -48,7 +48,7 @@ class ht:
   int32 = strat.integers(-2147483648, 2147483647)
   int64 = strat.integers(-9223372036854775808, 9223372036854775807)
   bool = strat.booleans()
-ht.bfloat16 = ht.uint16.filter(lambda x: ((x >> 7) & 0xFF) != 0)  # filter subnormal bfloat16
+ht.bfloat16 = ht.uint16
 ht.fp8e4m3 = ht.uint8
 ht.fp8e5m2 = ht.uint8
 
@@ -138,6 +138,7 @@ class TestDTypeALU(unittest.TestCase):
   def test_float16_unary(self, a, op): universal_test_unary(a, dtypes.float16, op)
 
   @unittest.skipUnless(is_dtype_supported(dtypes.bfloat16), f"no bfloat16 on {Device.DEFAULT}")
+  @unittest.skipIf(CPU_LLVM, "bfloat16 precision issues with CPU_LLVM")
   @given(ht.bfloat16, strat.sampled_from(unary_operations))
   def test_bfloat16_unary(self, a, op): universal_test_unary(from_storage_scalar(a, dtypes.bfloat16), dtypes.bfloat16, op)
 

test/test_image_dtype.py

@@ -44,66 +44,6 @@ class TestImageCopy(unittest.TestCase):
 
 @unittest.skipUnless(REAL_DEV in IMAGE_SUPPORTED_DEVICES, "Images not supported")
 class TestImageDType(unittest.TestCase):
-  def test_image_pitch(self):
-    def __validate(imgdt, expected_pitch):
-      assert imgdt.pitch == expected_pitch, f"Failed pitch for image: {imgdt}. Got 0x{imgdt.pitch:X}, expected 0x{expected_pitch:X}"
-
-    # Match opencl pitches for perf
-    __validate(dtypes.imageh((1, 201)), 0x680)
-    __validate(dtypes.imageh((16, 216)), 0x700)
-    __validate(dtypes.imageh((16, 9)), 0x80)
-    __validate(dtypes.imageh((48, 64)), 0x200)
-    __validate(dtypes.imageh((32, 128)), 0x400)
-    __validate(dtypes.imageh((96, 128)), 0x400)
-    __validate(dtypes.imageh((64, 256)), 0x840)
-    __validate(dtypes.imageh((64, 9)), 0x80)
-    __validate(dtypes.imageh((192, 256)), 0x840)
-    __validate(dtypes.imageh((64, 768)), 0x1840)
-    __validate(dtypes.imageh((256, 49)), 0x1C0)
-    __validate(dtypes.imageh((128, 9)), 0x80)
-    __validate(dtypes.imageh((16, 1024)), 0x2080)
-    __validate(dtypes.imageh((64, 512)), 0x1040)
-    __validate(dtypes.imageh((16, 512)), 0x1080)
-    __validate(dtypes.imageh((132, 64)), 0x200)
-    __validate(dtypes.imageh((4, 512)), 0x1200)
-    __validate(dtypes.imageh((8, 512)), 0x1100)
-    __validate(dtypes.imageh((128, 128)), 0x400)
-    __validate(dtypes.imageh((32, 512)), 0x1040)
-    __validate(dtypes.imageh((26, 64)), 0x200)
-    __validate(dtypes.imageh((32, 516)), 0x1040)
-    __validate(dtypes.imageh((32, 1024)), 0x2040)
-    __validate(dtypes.imageh((16, 2048)), 0x4080)
-    __validate(dtypes.imageh((8, 2048)), 0x4100)
-    __validate(dtypes.imageh((4, 4096)), 0x8200)
-
-    __validate(dtypes.imagef((16, 49)), 0x380)
-    __validate(dtypes.imagef((16, 1024)), 0x4080)
-    __validate(dtypes.imagef((256, 64)), 0x400)
-    __validate(dtypes.imagef((64, 512)), 0x2040)
-    __validate(dtypes.imagef((16, 512)), 0x2080)
-    __validate(dtypes.imagef((132, 64)), 0x400)
-    __validate(dtypes.imagef((4, 512)), 0x2200)
-    __validate(dtypes.imagef((4, 16)), 0x200)
-    __validate(dtypes.imagef((2, 16)), 0x400)
-    __validate(dtypes.imagef((8, 512)), 0x2100)
-    __validate(dtypes.imagef((12, 64)), 0x400)
-    __validate(dtypes.imagef((3, 32)), 0x400)
-    __validate(dtypes.imagef((128, 128)), 0x840)
-    __validate(dtypes.imagef((32, 512)), 0x2040)
-    __validate(dtypes.imagef((8, 3072)), 0xC100)
-    __validate(dtypes.imagef((4, 2048)), 0x8200)
-    __validate(dtypes.imagef((4, 1024)), 0x4200)
-    __validate(dtypes.imagef((4, 4096)), 0x10200)
-    __validate(dtypes.imagef((10, 384)), 0x1900)
-    __validate(dtypes.imagef((24, 64)), 0x400)
-    __validate(dtypes.imagef((128, 12)), 0xC0)
-    __validate(dtypes.imagef((10, 24)), 0x200)
-    __validate(dtypes.imagef((1, 129)), 0x840)
-    __validate(dtypes.imagef((1, 32)), 0x200)
-    __validate(dtypes.imagef((1, 64)), 0x400)
-    __validate(dtypes.imagef((1, 1239)), 0x4D80)
-    __validate(dtypes.imagef((1, 1)), 0x40)
-
   def test_image_and_back(self):
     data = Tensor.randn(9*27*4).realize()
     tst = data.numpy()

test/test_multitensor.py

@@ -256,11 +256,6 @@ class TestMultiTensor(unittest.TestCase):
       a,b = _test_allreduce(Tensor.rand(256, 256))
       np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5)
 
-  def test_allreduce_all2all(self):
-    with Context(ALL2ALL=2):
-      a,b = _test_allreduce(Tensor.rand(256, 256))
-      np.testing.assert_almost_equal(a.numpy(), b.numpy(), decimal=5)
-
   def test_copy_jit(self):
     @TinyJit
     def copy_tensor(x:Tensor): return (x.to(f"{x.device.split(':')[0]}:1") + 1)

test/test_ops.py

@@ -848,7 +848,7 @@ class TestOps(unittest.TestCase):
     helper_test_op([(45,65)], lambda x: x.cos())
     helper_test_op([()], lambda x: x.cos())
     if not ((getenv("MOCKGPU") and Device.DEFAULT == "NV") or Device.DEFAULT == "WEBGPU"):
-      helper_test_op(None, lambda x: x.cos(), vals=[[math.nan, math.inf, -math.inf, 0.0]])
+      helper_test_op(None, lambda x: x.sin(), vals=[[math.nan, math.inf, -math.inf, 0.0]])
       helper_test_op(None, lambda x: x.cos(), vals=[[1e1, 1e2, 1e3, 1e4, 1e5, 1e6, -1e1, -1e2, -1e3, -1e4, -1e5, -1e6]],
                     atol=3e-3, rtol=3e-3, grad_atol=3e-3, grad_rtol=3e-3)
   @unittest.skipIf(Device.DEFAULT == "WEBGPU" and platform.system() == "Windows", "Not accurate enough with DirectX backend")
@@ -859,8 +859,8 @@ class TestOps(unittest.TestCase):
     helper_test_op([(45,65)], lambda x: x.tan(), low=-5, high=5)
     helper_test_op([()], lambda x: x.tan())
     if not ((getenv("MOCKGPU") and Device.DEFAULT == "NV") or Device.DEFAULT == "WEBGPU"):
-      helper_test_op(None, lambda x: x.tan(), vals=[[math.nan, math.inf, -math.inf, 0.0]])
-      helper_test_op(None, lambda x: x.tan(), vals=[[1e1, 1e2, 1e3, 1e4, 1e5, 1e6, -1e1, -1e2, -1e3, -1e4, -1e5, -1e6]],
+      helper_test_op(None, lambda x: x.sin(), vals=[[math.nan, math.inf, -math.inf, 0.0]])
+      helper_test_op(None, lambda x: x.cos(), vals=[[1e1, 1e2, 1e3, 1e4, 1e5, 1e6, -1e1, -1e2, -1e3, -1e4, -1e5, -1e6]],
                     atol=3e-3, rtol=3e-3, grad_atol=3e-3, grad_rtol=3e-3)
 
   def test_asin(self):
@@ -1655,7 +1655,7 @@ class TestOps(unittest.TestCase):
   def test_broadcast_full(self):
     for torch_op, tinygrad_op in [(torch.add, Tensor.add), (torch.sub, Tensor.sub), (torch.mul, Tensor.mul),
                                   (torch.div, Tensor.div), (torch.pow, Tensor.pow)]:
-      for shapes in [((5,3,14,16), (5,1,14,1)), ((1,3,1,7,1), (2,1,5,1,8))]:
+      for shapes in [((5,13,24,16), (5,1,24,1)), ((1,3,1,7,1), (2,1,5,1,8))]:
         with self.subTest(op=torch_op.__name__, shapes=shapes):
           if tinygrad_op != Tensor.pow:
             helper_test_op(shapes, torch_op, tinygrad_op)
@@ -2078,7 +2078,7 @@ class TestOps(unittest.TestCase):
       lambda x,w: Tensor.conv2d(x,w,padding=[1,1,1,1,1,1]), grad_rtol=1e-5)
 
   def test_simple_conv2d_m4(self):
-    helper_test_op([(1,16,9,9), (16,16,3,3)],
+    helper_test_op([(1,16,18,18), (16,16,3,3)],
       lambda x,w: torch.nn.functional.conv2d(x,w),
       lambda x,w: Tensor.conv2d(x,w), atol=1e-05, grad_rtol=1e-5)
 
@@ -2535,7 +2535,7 @@ class TestOps(unittest.TestCase):
 
   @slow_test
   def test_avg_pool2d(self):
-    shape = (32,2,11,28)
+    shape = (32,2,111,28)
     for ksz in [(2,2), (3,3), (3,2), (5,5), (5,1)]:
       with self.subTest(kernel_size=ksz):
         helper_test_op([shape],
@@ -2549,7 +2549,7 @@ class TestOps(unittest.TestCase):
 
   @slow_test
   def test_avg_pool2d_padding(self):
-    shape = (32,2,11,28)
+    shape = (32,2,111,28)
     for ksz in [(2,2), (3,3), 2, 3, (3,2)]:
       for p in [1, (1,0), (0,1)]:
         with self.subTest(kernel_size=ksz, padding=p):
@@ -2557,10 +2557,10 @@ class TestOps(unittest.TestCase):
             lambda x: torch.nn.functional.avg_pool2d(x, kernel_size=ksz, padding=p),
             lambda x: Tensor.avg_pool2d(x, kernel_size=ksz, padding=p), rtol=1e-5)
     with self.assertRaises(ValueError):
-      Tensor.avg_pool2d(Tensor.randn((32,2,11,28)), kernel_size=(2,2), padding=(1,1,1))
+      Tensor.avg_pool2d(Tensor.randn((32,2,111,28)), kernel_size=(2,2), padding=(1,1,1))
 
   def test_avg_pool2d_asymmetric_padding(self):
-    shape = (32,2,11,28)
+    shape = (32,2,111,28)
     for p in [(0,1,0,1), (2,1,2,1), (2,0,2,1)]:
       with self.subTest(padding=p):
         helper_test_op([shape],
@@ -2571,7 +2571,7 @@ class TestOps(unittest.TestCase):
 
   @slow_test
   def test_avg_pool2d_padding_not_counted(self):
-    shape = (32,2,11,28)
+    shape = (32,2,111,28)
     for ksz in [(2,2), (3,3), 2, 3, (3,2)]:
       with self.subTest(kernel_size=ksz):
         helper_test_op([shape],
@@ -2607,9 +2607,9 @@ class TestOps(unittest.TestCase):
       lambda x: Tensor.avg_pool2d(x, kernel_size=(3,3), stride=3, padding=1, ceil_mode=True, count_include_pad=True))
 
   def test_global_avg_pool2d(self):
-    helper_test_op([(32,2,11,28)],
-      lambda x: torch.nn.functional.avg_pool2d(x, kernel_size=(11,28)),
-      lambda x: Tensor.avg_pool2d(x, kernel_size=(11,28)), rtol=1e-5)
+    helper_test_op([(32,2,111,28)],
+      lambda x: torch.nn.functional.avg_pool2d(x, kernel_size=(111,28)),
+      lambda x: Tensor.avg_pool2d(x, kernel_size=(111,28)), rtol=1e-5)
 
   def test_avg_pool3d(self):
     # TODO: AMD_LLVM has larger atol
@@ -3142,10 +3142,10 @@ class TestOps(unittest.TestCase):
         lambda x: x.log_softmax(axis=1).nll_loss(Tensor(target), Tensor(weight), reduction=r))
 
   def test_nll_loss_3d_weight(self):
-    target = np.random.randint(0, 10, (16,3,3,3), dtype=np.int32).tolist()
+    target = np.random.randint(0, 10, (32,3,3,3), dtype=np.int32).tolist()
     weight = np.random.normal(0, 1, (10,)).astype(np.float32).tolist()
     for r in ("mean", "sum", "none"):
-      helper_test_op([(16,10,3,3,3)],
+      helper_test_op([(32,10,3,3,3)],
           lambda x: torch.nn.functional.nll_loss(torch.nn.functional.log_softmax(x, dim=1), torch.tensor(target), torch.tensor(weight), reduction=r),
           lambda x: x.log_softmax(axis=1).nll_loss(Tensor(target), Tensor(weight), reduction=r))
 

test/test_uop_graph.py

@@ -478,6 +478,143 @@ class TestUOpGraph(unittest.TestCase):
     for u in uops:
       self.assertNotEqual(u.dtype, dtypes.long)
 
+  def test_in_out_of_bounds_access(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(dtypes.int, 0), ptr=True),))
+      to_uops_list([ld0])
+      ld1 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(dtypes.int, 15), ptr=True),))
+      to_uops_list([ld1])
+      ld1 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(dtypes.int, 7), ptr=True),))
+      to_uops_list([ld1])
+
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(dtypes.int, 42), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld0])
+
+  def test_in_out_of_bounds_access_symbolic(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(Variable("i", 1, 10), ptr=True),))
+      to_uops_list([ld0])
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(Variable("i", 0, 15), ptr=True),))
+      to_uops_list([ld0])
+
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(Variable("i", 0, 20), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld0])
+
+  def test_in_out_of_bounds_access_gated_store(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), src=(), arg=0)
+      v = Variable("v", 0, 20)
+      st0 = UOp(Ops.STORE, dtypes.void, src=(glbl0.index(v.valid(v<16)), UOp.const(dtypes.int, 0)))
+      to_uops_list([st0])
+
+      st1 = UOp(Ops.STORE, dtypes.void, (glbl0.index(v.valid(v<20)), v))
+      with self.assertRaises(RuntimeError): to_uops_list([st1])
+
+  @unittest.skip("if not allowed in graph")
+  def test_in_bounds_access_gated_local(self):
+    with Context(IGNORE_OOB=0):
+      # Define buffers
+      gbuf = UOp(Ops.DEFINE_GLOBAL, dtypes.uint.ptr(400), (), 0)
+      sbuf = UOp(Ops.DEFINE_LOCAL, dtypes.uint.ptr(8, addrspace=AddrSpace.LOCAL), (), "temp0")
+
+      # Define indices, valids and barrier
+      gidx = UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 416),), "gidx0")
+      lidx = UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 10),), "lidx0")
+
+      gate = (gidx<400) & (lidx<8)
+
+      local_store = UOp(Ops.STORE, dtypes.void, (sbuf.index(lidx, lidx<8), UOp.const(dtypes.uint, 1)))
+
+      barrier = UOp(Ops.BARRIER, dtypes.void, (local_store,))
+      if_barrier = UOp(Ops.IF, dtypes.void, (gate, barrier))
+
+      # Load from local memory (after the IF/barrier)
+      local_load = UOp(Ops.LOAD, dtypes.uint, (sbuf.index(lidx, ptr=True), if_barrier))
+
+      # Store to global memory
+      global_store = UOp(Ops.STORE, dtypes.void, (gbuf.index(gidx), local_load))
+      to_uops_list([global_store])
+
+  def test_load_with_float_in_index(self):
+    with Context(IGNORE_OOB=0):
+      ridx = UOp.range(20, 0)
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      i = (ridx.cast(dtypes.float)*0.68).trunc().cast(dtypes.int)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(i.valid((0<=i)&(i<16)), ptr=True),))
+      to_uops_list([ld0])
+      glblfloat = UOp(Ops.DEFINE_GLOBAL, dtypes.float.ptr(20), (), 0)
+      ldfloat = UOp(Ops.LOAD, dtypes.float, (glblfloat.index(ridx),))
+      i = (ldfloat+3.14).cast(dtypes.int)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(i, ((0<=i)&(i<16)), ptr=True),))
+
+  def test_load_cast_to_bool(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(1), (), 0)
+      ridx = UOp.range(20, 0)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(ridx.valid(ridx.cast(dtypes.bool).logical_not()), ptr=True),))
+      to_uops_list([ld0])
+
+  @unittest.skip("Bool load is not supported yet")
+  def test_load_mask(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      mask = UOp(Ops.DEFINE_GLOBAL, dtypes.bool.ptr(16), (), 0)
+      ridx = UOp.range(20, 0)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(ridx, ridx<16&mask), ptr=True)))
+      to_uops_list([ld0])
+
+  def test_out_of_bounds_off_by_one_access(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(dtypes.int, 16), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld0])
+
+  def test_in_out_bounds_access_with_mask(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      gidx0 = UOp.range(42, 0, AxisType.GLOBAL)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(gidx0.valid((5<gidx0)&(gidx0<16)), ptr=True),))
+      ld1 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(gidx0.valid(gidx0<16), ptr=True),))
+      to_uops_list([ld0, ld1])
+
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(gidx0.valid(gidx0<17), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld0])
+
+  def test_in_out_of_bounds_access_symbolic_mask(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      i = Variable("i", 1, 80)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(i.valid(i<10), ptr=True),))
+      to_uops_list([ld0])
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(i.valid(i<15), ptr=True),))
+      to_uops_list([ld0])
+
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(i.valid(i<20), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld0])
+
+  def test_in_out_of_bounds_access_index_load(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
+      glbl1 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(64), (), 0)
+      gidx0 = UOp.range(42, 0, AxisType.GLOBAL)
+      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(gidx0.valid(gidx0<8), ptr=True),)).cast(dtypes.index)
+      ld1 = UOp(Ops.LOAD, dtypes.int, (glbl1.index((ld0*2).valid((ld0>=0)&(ld0<32)), ptr=True),))
+      to_uops_list([ld1])
+
+      ld1 = UOp(Ops.LOAD, dtypes.int, (glbl1.index((ld0*2).valid((ld0>=0)&(ld0<64)), ptr=True),))
+      with self.assertRaises(RuntimeError): to_uops_list([ld1])
+
+  def test_bounds_with_loaded_bool(self):
+    with Context(IGNORE_OOB=0):
+      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.bool.ptr(16), (), 0)
+      glbl1 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(8), (), 0)
+      gidx0 = UOp(Ops.SPECIAL, dtypes.index, (UOp.const(dtypes.index, 16),), "gidx0")
+      ld0 = glbl0.index(gidx0, ptr=True).load()
+      ld1 = glbl1.index(gidx0.valid(ld0), ptr=True).load()
+      with self.assertRaises(RuntimeError): to_uops_list([ld1])
+
   def test_fold_gated_load(self):
     glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(), (), 0)
     glbl1 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(), (), 1)

test/testextra/test_cfg_viz.py

@@ -10,7 +10,7 @@ from tinygrad.renderer import ProgramSpec
 from tinygrad.helpers import TracingKey, getenv
 from tinygrad.engine.realize import ExecItem, CompiledRunner
 
-from extra.assembly.amd.autogen.rdna3.ins import *
+from extra.assembly.amd.autogen.rdna3 import *
 
 # TODO: use the RDNA3 renderer when it's in master
 template = """.text

test/unit/test_validate_oob.py

@@ -1,179 +0,0 @@
-import unittest
-from tinygrad import dtypes, Variable
-from tinygrad.dtype import AddrSpace
-from tinygrad.helpers import Context
-from tinygrad.uop.ops import Ops, UOp, AxisType
-from test.test_uops import to_uops_list
-
-class TestValidateOOB(unittest.TestCase):
-  """Test z3 validation of index bounds for different ALU ops and patterns."""
-
-  # basic index patterns
-  def test_const_index(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      to_uops_list([buf.index(UOp.const(dtypes.int, 0), ptr=True).load(dtype=dtypes.int)])  # valid
-      to_uops_list([buf.index(UOp.const(dtypes.int, 15), ptr=True).load(dtype=dtypes.int)])  # valid (last element)
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(UOp.const(dtypes.int, 16), ptr=True).load(dtype=dtypes.int)])  # off by one
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(UOp.const(dtypes.int, 42), ptr=True).load(dtype=dtypes.int)])  # way out
-
-  def test_variable_index(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      to_uops_list([buf.index(Variable("i", 0, 15), ptr=True).load(dtype=dtypes.int)])  # valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(Variable("i", 0, 20), ptr=True).load(dtype=dtypes.int)])  # oob
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(Variable("i", -5, 10), ptr=True).load(dtype=dtypes.int)])  # negative
-
-  def test_range_with_mask(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      r = UOp.range(42, 0, AxisType.GLOBAL)
-      to_uops_list([buf.index(r.valid(r < 16), ptr=True).load(dtype=dtypes.int)])  # valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(r.valid(r < 17), ptr=True).load(dtype=dtypes.int)])  # oob
-
-  def test_variable_with_mask(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      v = Variable("v", -5, 80)
-      to_uops_list([buf.index(v.valid((v >= 0) & (v < 16)), ptr=True).load(dtype=dtypes.int)])  # valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(v.valid(v < 20), ptr=True).load(dtype=dtypes.int)])  # negative not masked
-
-  def test_gated_store(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      v = Variable("v", 0, 20)
-      to_uops_list([buf.index(v.valid(v < 16)).store(0)])  # valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(v.valid(v < 20)).store(0)])  # oob
-
-  # ALU ops in index
-  def test_idiv(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      to_uops_list([buf.index(UOp.range(32, 0, AxisType.GLOBAL) // 2, ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(UOp.range(34, 0, AxisType.GLOBAL) // 2, ptr=True).load(dtype=dtypes.int)])  # 0..16 oob
-
-  def test_mod(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      r = UOp.range(100, 0, AxisType.GLOBAL)
-      to_uops_list([buf.index(r % 16, ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(r % 20, ptr=True).load(dtype=dtypes.int)])  # 0..19 oob
-
-  def test_shr(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      to_uops_list([buf.index(UOp.range(64, 0, AxisType.GLOBAL) >> 2, ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(UOp.range(128, 0, AxisType.GLOBAL) >> 2, ptr=True).load(dtype=dtypes.int)])  # 0..31 oob
-
-  def test_shl(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(64), (), 0)
-      r = UOp.range(8, 0, AxisType.GLOBAL)
-      to_uops_list([buf.index(r << 2, ptr=True).load(dtype=dtypes.int)])  # 0..28 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(r << 4, ptr=True).load(dtype=dtypes.int)])  # 0..112 oob
-
-  def test_and(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      r = UOp.range(100, 0, AxisType.GLOBAL)
-      to_uops_list([buf.index(r & 15, ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(r & 31, ptr=True).load(dtype=dtypes.int)])  # 0..31 oob
-
-  def test_max(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      to_uops_list([buf.index(Variable("v", -10, 15).maximum(0), ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(Variable("v2", -10, 20).maximum(0), ptr=True).load(dtype=dtypes.int)])  # 0..20 oob
-
-  def test_xor_in_mask(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      r = UOp.range(32, 0, AxisType.GLOBAL)
-      to_uops_list([buf.index(r.valid((r < 8) ^ ((r >= 8) & (r < 16))), ptr=True).load(dtype=dtypes.int)])  # 0..15 valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf.index(r.valid((r < 10) ^ (r >= 20)), ptr=True).load(dtype=dtypes.int)])  # 0..9,20..31 oob
-
-  # cast patterns
-  def test_float_cast_in_index(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      r = UOp.range(20, 0)
-      i = (r.cast(dtypes.float) * 0.68).trunc().cast(dtypes.int)
-      to_uops_list([buf.index(i.valid((i >= 0) & (i < 16)), ptr=True).load(dtype=dtypes.int)])
-
-  def test_bool_cast_in_mask(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(1), (), 0)
-      r = UOp.range(20, 0)
-      to_uops_list([buf.index(r.valid(r.cast(dtypes.bool).logical_not()), ptr=True).load(dtype=dtypes.int)])  # only r=0 valid
-
-  # load result as index/mask
-  def test_load_as_index(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      buf1 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(64), (), 1)
-      r = UOp.range(42, 0, AxisType.GLOBAL)
-      ld0 = buf0.index(r.valid(r < 8), ptr=True).load(dtype=dtypes.int).cast(dtypes.index)
-      to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 32)), ptr=True).load(dtype=dtypes.int)])  # valid
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf1.index((ld0 * 2).valid((ld0 >= 0) & (ld0 < 64)), ptr=True).load(dtype=dtypes.int)])  # oob
-
-  def test_load_bool_as_mask(self):
-    with Context(IGNORE_OOB=0, SPEC=2):
-      buf_bool = UOp(Ops.DEFINE_GLOBAL, dtypes.bool.ptr(16), (), 0)
-      buf_int = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(8), (), 1)
-      gidx = UOp(Ops.SPECIAL, dtypes.index, (UOp.const(dtypes.index, 16),), "gidx0")
-      ld_bool = buf_bool.index(gidx, ptr=True).load()
-      with self.assertRaises(RuntimeError):
-        to_uops_list([buf_int.index(gidx.valid(ld_bool), ptr=True).load()])  # gidx 0..15, buf_int size 8
-
-  # skipped tests (moved from test_uop_graph.py)
-  @unittest.skip("if not allowed in graph")
-  def test_in_bounds_access_gated_local(self):
-    with Context(IGNORE_OOB=0):
-      # Define buffers
-      gbuf = UOp(Ops.DEFINE_GLOBAL, dtypes.uint.ptr(400), (), 0)
-      sbuf = UOp(Ops.DEFINE_LOCAL, dtypes.uint.ptr(8, addrspace=AddrSpace.LOCAL), (), "temp0")
-
-      # Define indices, valids and barrier
-      gidx = UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 416),), "gidx0")
-      lidx = UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 10),), "lidx0")
-
-      gate = (gidx<400) & (lidx<8)
-
-      local_store = UOp(Ops.STORE, dtypes.void, (sbuf.index(lidx, lidx<8), UOp.const(dtypes.uint, 1)))
-
-      barrier = UOp(Ops.BARRIER, dtypes.void, (local_store,))
-      if_barrier = UOp(Ops.IF, dtypes.void, (gate, barrier))
-
-      # Load from local memory (after the IF/barrier)
-      local_load = UOp(Ops.LOAD, dtypes.uint, (sbuf.index(lidx, ptr=True), if_barrier))
-
-      # Store to global memory
-      global_store = UOp(Ops.STORE, dtypes.void, (gbuf.index(gidx), local_load))
-      to_uops_list([global_store])
-
-  @unittest.skip("Bool load is not supported yet")
-  def test_load_mask(self):
-    with Context(IGNORE_OOB=0):
-      glbl0 = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(16), (), 0)
-      mask = UOp(Ops.DEFINE_GLOBAL, dtypes.bool.ptr(16), (), 0)
-      ridx = UOp.range(20, 0)
-      ld0 = UOp(Ops.LOAD, dtypes.int, (glbl0.index(UOp.const(ridx, ridx<16&mask), ptr=True)))
-      to_uops_list([ld0])
-
-if __name__ == "__main__":
-  unittest.main()

tinygrad/dtype.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 from typing import Final, ClassVar, Callable, Literal
 import math, struct, ctypes, functools
 from dataclasses import dataclass, fields
-from tinygrad.helpers import getenv, prod, round_up, next_power2
+from tinygrad.helpers import getenv, prod
 from enum import Enum, auto
 
 class InvalidTypeMetaClass(type):
@@ -101,15 +101,6 @@ class ImageDType(PtrDType):
     assert addrspace == AddrSpace.GLOBAL, "images can't be local"
     return self
   def __repr__(self): return f"dtypes.{self.name}({self.shape})" + (f'.vec({self.v})' if self.v != 1 else '')
-  @property
-  def pitch(self):
-    imgw, imgh, itemsize_log = self.shape[1], self.shape[0], int(math.log2(self.itemsize))
-    pitchalign = max(6, 11 - int(math.log2(imgh))) if imgh > 1 else 6
-    align_up = max(1, (8 // itemsize_log + 1) - imgh // 32) if pitchalign == 6 else (2 ** (pitchalign - itemsize_log - 2))
-
-    granularity = 128 if self.itemsize == 4 else 256
-    pitch_add = (1 << pitchalign) if min(next_power2(imgw), round_up(imgw, granularity)) - align_up + 1 <= imgw and imgw > granularity//2 else 0
-    return round_up(imgw * 4 * self.itemsize, 1 << pitchalign) + pitch_add
 
 class dtypes:
   @staticmethod

tinygrad/helpers.py

@@ -50,7 +50,6 @@ def strip_parens(fst:str) -> str: return fst[1:-1] if fst and fst[0]=='(' and fs
 def ceildiv(num, amt): return int(ret) if isinstance((ret:=-(num//-amt)), float) else ret
 def round_up(num:int, amt:int) -> int: return (num+amt-1)//amt * amt
 def round_down(num:int, amt:int) -> int: return -round_up(-num, amt)
-def next_power2(x): return 1 if x == 0 else 1 << (x - 1).bit_length()
 # cstyle div and mod
 def cdiv(x:int, y:int) -> int: return abs(x)//abs(y)*(1,-1)[x*y<0] if y != 0 else 0
 def cmod(x:int, y:int) -> int: return x-cdiv(x,y)*y
@@ -182,8 +181,8 @@ JIT, JIT_BATCH_SIZE = ContextVar("JIT", 2 if OSX and ARCH_X86 else 1), ContextVa
 WINO, CAPTURING, TRACEMETA = ContextVar("WINO", 0), ContextVar("CAPTURING", 1), ContextVar("TRACEMETA", 1)
 USE_TC, TC_SELECT, TC_OPT, AMX = ContextVar("TC", 1), ContextVar("TC_SELECT", -1), ContextVar("TC_OPT", 0), ContextVar("AMX", 0)
 TRANSCENDENTAL, NOLOCALS = ContextVar("TRANSCENDENTAL", 1), ContextVar("NOLOCALS", 0)
-SPLIT_REDUCEOP, NO_MEMORY_PLANNER, LRU = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("LRU", 1)
-RING, ALL2ALL = ContextVar("RING", 1), ContextVar("ALL2ALL", 0)
+SPLIT_REDUCEOP, NO_MEMORY_PLANNER, RING = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("RING", 1)
+LRU = ContextVar("LRU", 1)
 CACHELEVEL, IGNORE_BEAM_CACHE, DEVECTORIZE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0), ContextVar("DEVECTORIZE", 1)
 VALIDATE_WITH_CPU, DISABLE_FAST_IDIV = ContextVar("VALIDATE_WITH_CPU", 0), ContextVar("DISABLE_FAST_IDIV", 0)
 CORRECT_DIVMOD_FOLDING, FUSE_OPTIM = ContextVar("CORRECT_DIVMOD_FOLDING", 0), ContextVar("FUSE_OPTIM", 0)

tinygrad/runtime/graph/hcq.py

@@ -1,6 +1,6 @@
-import collections, itertools, time
+import collections, time
 from typing import Any, cast
-from tinygrad.helpers import round_up, PROFILE, ALL2ALL, merge_dicts, getenv, dedup, suppress_finalizing
+from tinygrad.helpers import round_up, PROFILE, merge_dicts, getenv, dedup, suppress_finalizing
 from tinygrad.runtime.support.hcq import HCQCompiled, HCQAllocator, HCQSignal, HCQBuffer, HWQueue, HCQArgsState, BumpAllocator, MMIOInterface
 from tinygrad.device import Buffer, BufferSpec, Compiled, Device, ProfileGraphEntry, ProfileGraphEvent
 from tinygrad.dtype import dtypes
@@ -22,7 +22,7 @@ class HCQGraph(MultiGraphRunner):
 
     for (j,i), input_idx in self.input_replace.items():
       x = self.input_replace_to_var.setdefault((j,i), UOp.variable(f"input_{input_idx}", 0, 0xffffffffffffffff, dtype=dtypes.uint64))
-      self.hcq_bufs[j][i] = HCQBuffer(x, self.hcq_bufs[j][i].size, image=self.hcq_bufs[j][i].image) # Create fake buffer with variable
+      self.hcq_bufs[j][i] = HCQBuffer(x, self.hcq_bufs[j][i].size, texture_info=self.hcq_bufs[j][i].texture_info) # Create fake buffer with variable
 
     # Allocate kernel args.
     kernargs_size: dict[Compiled, int] = collections.defaultdict(int)
@@ -49,9 +49,7 @@ class HCQGraph(MultiGraphRunner):
     self.ji_schedule: dict[int, tuple[HCQCompiled, HWQueue, list, list, HCQSignal, int|None]] = {}
 
     self.comp_queues: dict[HCQCompiled, HWQueue] = {dev: dev.hw_compute_queue_t() for dev in self.devices}
-    self.copy_queues: dict[tuple[HCQCompiled, int], HWQueue] = {} # lazy allocation, keyed by (device, queue_idx)
-    self.num_copy_queues: int = getenv("HCQ_NUM_SDMA", 2 if ALL2ALL >= 1 else 1)
-    self.copy_queue_cnt: collections.defaultdict[HCQCompiled, itertools.count] = collections.defaultdict(itertools.count)
+    self.copy_queues: dict[HCQCompiled, HWQueue] = {} # lazy allocation
 
     self.signals: dict[Any, HCQSignal] = {**{dev: dev.new_signal(value=0) for dev in self.devices if not dev._is_cpu()},
       **{"KICK": self.devices[0].new_signal(value=0)}, **{dev: self.devices[0].new_signal(value=0) for dev in self.devices if dev._is_cpu()}}
@@ -87,8 +85,7 @@ class HCQGraph(MultiGraphRunner):
         enqueue_queue = self.comp_queues[enqueue_dev]
       else:
         assert (enqueue_dev.hw_copy_queue_t is not None), "device must implement a copy queue"
-        queue_idx = next(self.copy_queue_cnt[enqueue_dev]) % self.num_copy_queues
-        enqueue_queue = self.copy_queues.setdefault((enqueue_dev, queue_idx), enqueue_dev.hw_copy_queue_t(queue_idx=queue_idx))
+        enqueue_queue = self.copy_queues.setdefault(enqueue_dev, enqueue_dev.hw_copy_queue_t())
 
       out_signal = self.signals.setdefault(enqueue_queue, self.devices[0].new_signal(value=0))
 
@@ -178,17 +175,14 @@ class HCQGraph(MultiGraphRunner):
 
     for dev in self.devices:
       for dep_dev in list(self.copy_to_devs[dev]) + [dev]:
-        for copy_q in self._dev_copy_queues(dep_dev):
-          if copy_q in self.signals: self.comp_queues[dev].wait(self.signals[copy_q], cast(int, last_j[copy_q]) + 1)
+        if dep_dev in self.copy_queues: self.comp_queues[dev].wait(self.signals[(copy_q:=self.copy_queues[dep_dev])], cast(int, last_j[copy_q]) + 1)
 
       self.comp_queues[dev].signal(self.virt_timeline_signals[dev], self.virt_timeline_vals[dev] + 1).bind(dev)
-      for copy_q in self._dev_copy_queues(dev): copy_q.bind(dev)
+      if dev in self.copy_queues: self.copy_queues[dev].bind(dev)
 
     self.last_timeline: dict[HCQCompiled, tuple[HCQSignal, int]] = {dev: (dev.timeline_signal, 0) for dev in self.devices}
     self.queue_signals_to_reset = [self.signals[q] for q in list(self.comp_queues.values()) + list(self.copy_queues.values()) if q in self.signals]
 
-  def _dev_copy_queues(self, dev): return [q for (d, _), q in self.copy_queues.items() if d == dev]
-
   def __call__(self, input_rawbuffers: list[Buffer], var_vals: dict[str, int], wait=False) -> float|None:
     # Wait and restore signals
     self.kickoff_value += 1
@@ -211,7 +205,8 @@ class HCQGraph(MultiGraphRunner):
 
     for dev in self.devices:
       self.comp_queues[dev].submit(dev, hcq_var_vals_local:=hcq_var_vals|self.fixedvars.get(dev, {}))
-      for copy_queue in self._dev_copy_queues(dev): copy_queue.submit(dev, hcq_var_vals_local)
+      if (copy_queue:=self.copy_queues.get(dev, None)) is not None: copy_queue.submit(dev, hcq_var_vals_local)
+
       self.last_timeline[dev] = (dev.timeline_signal, dev.next_timeline())
 
     if wait:

tinygrad/runtime/ops_amd.py

@@ -446,8 +446,8 @@ class AMDComputeAQLQueue(AMDComputeQueue):
     dev.compute_queue.signal_doorbell(dev, doorbell_value=dev.compute_queue.put_value-1)
 
 class AMDCopyQueue(HWQueue):
-  def __init__(self, dev, max_copy_size=0x40000000, queue_idx=0):
-    self.dev, self.sdma, self.internal_cmd_sizes, self.max_copy_size, self.queue_idx = dev, dev.sdma, [], max_copy_size, queue_idx
+  def __init__(self, dev, max_copy_size=0x40000000):
+    self.dev, self.sdma, self.internal_cmd_sizes, self.max_copy_size = dev, dev.sdma, [], max_copy_size
     super().__init__()
 
   def q(self, *arr):
@@ -501,42 +501,41 @@ class AMDCopyQueue(HWQueue):
     self._q, self.cmd_sizes = hw_view, [len(self.indirect_cmd)]
 
   def _submit(self, dev:AMDDevice):
-    sdma_queue = dev.sdma_queue(self.queue_idx)
     if self.binded_device == dev:
       # An IB packet must end on a 8 DW boundary.
-      add = (8 - (((sdma_queue.put_value % 32) // 4) + len(self.indirect_cmd) % 8)) % 8
+      add = (8 - (((dev.sdma_queue.put_value % 32) // 4) + len(self.indirect_cmd) % 8)) % 8
       cmds, cmd_sizes = ([0] * add) + self.indirect_cmd, [len(self.indirect_cmd) + add]
 
-      if len(cmds) * 4 >= (sdma_queue.ring.nbytes - sdma_queue.put_value % sdma_queue.ring.nbytes):
+      if len(cmds) * 4 >= (dev.sdma_queue.ring.nbytes - dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes):
         cmds, cmd_sizes = [0, 0] + self.indirect_cmd, [8]
     else: cmds, cmd_sizes = self._q, self.internal_cmd_sizes
 
     tail_blit_dword = 0
     for cmdsz in cmd_sizes:
-      if (tail_blit_dword + cmdsz) * 4 >= sdma_queue.ring.nbytes - sdma_queue.put_value % sdma_queue.ring.nbytes: break
+      if (tail_blit_dword + cmdsz) * 4 >= dev.sdma_queue.ring.nbytes - dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes: break
       tail_blit_dword += cmdsz
 
     # Force align of submits to hit our usb layer write cache.
     if (rem_packet_cnt := len(cmds) - tail_blit_dword) > 0 and dev.is_usb(): tail_blit_dword = 0
 
     # USB devices run in single-step mode, so they can't overrun the queue.
-    total_bytes = (tail_blit_dword * 4 if rem_packet_cnt == 0 else -sdma_queue.put_value % sdma_queue.ring.nbytes) + rem_packet_cnt * 4
-    assert total_bytes < sdma_queue.ring.nbytes, "SDMA queue overrun"
-    while not dev.is_usb() and sdma_queue.put_value + total_bytes - sdma_queue.read_ptr > sdma_queue.ring.nbytes: pass
+    total_bytes = (tail_blit_dword * 4 if rem_packet_cnt == 0 else -dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes) + rem_packet_cnt * 4
+    assert total_bytes < dev.sdma_queue.ring.nbytes, "SDMA queue overrun"
+    while not dev.is_usb() and dev.sdma_queue.put_value + total_bytes - dev.sdma_queue.read_ptr > dev.sdma_queue.ring.nbytes: pass
 
-    start_idx = (sdma_queue.put_value % sdma_queue.ring.nbytes) // 4
-    sdma_queue.ring[start_idx : start_idx + tail_blit_dword] = array.array('I', cmds[:tail_blit_dword])
-    sdma_queue.put_value += tail_blit_dword * 4
+    start_idx = (dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes) // 4
+    dev.sdma_queue.ring[start_idx : start_idx + tail_blit_dword] = array.array('I', cmds[:tail_blit_dword])
+    dev.sdma_queue.put_value += tail_blit_dword * 4
 
     if (rem_packet_cnt := len(cmds) - tail_blit_dword) > 0:
-      zero_fill = sdma_queue.ring.nbytes - sdma_queue.put_value % sdma_queue.ring.nbytes
-      sdma_queue.ring.view(sdma_queue.put_value % sdma_queue.ring.nbytes, zero_fill, fmt='B')[:] = bytes(zero_fill)
-      sdma_queue.put_value += zero_fill
+      zero_fill = dev.sdma_queue.ring.nbytes - dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes
+      dev.sdma_queue.ring.view(dev.sdma_queue.put_value % dev.sdma_queue.ring.nbytes, zero_fill, fmt='B')[:] = bytes(zero_fill)
+      dev.sdma_queue.put_value += zero_fill
 
-      sdma_queue.ring[0:rem_packet_cnt] = array.array('I', cmds[tail_blit_dword:])
-      sdma_queue.put_value += rem_packet_cnt * 4
+      dev.sdma_queue.ring[0:rem_packet_cnt] = array.array('I', cmds[tail_blit_dword:])
+      dev.sdma_queue.put_value += rem_packet_cnt * 4
 
-    sdma_queue.signal_doorbell(dev)
+    dev.sdma_queue.signal_doorbell(dev)
 
 class AMDProgram(HCQProgram):
   def __init__(self, dev:AMDDevice, name:str, lib:bytes):
@@ -757,8 +756,7 @@ class KFDIface:
     stm = kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(c_gpus), n_devices=1)
     assert stm.n_success == 1
 
-  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0,
-                   xcc_id=0, idx=0):
+  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0, xcc_id=0):
     queue = kfd.AMDKFD_IOC_CREATE_QUEUE(KFDIface.kfd, ring_base_address=ring.va_addr, ring_size=ring.size, gpu_id=self.gpu_id,
       queue_type=queue_type, queue_percentage=kfd.KFD_MAX_QUEUE_PERCENTAGE|(xcc_id<<8), queue_priority=getenv("AMD_KFD_QUEUE_PRIORITY", 7),
       eop_buffer_address=eop_buffer.va_addr if eop_buffer else 0, eop_buffer_size=eop_buffer.size if eop_buffer else 0, ctl_stack_size=ctl_stack_size,
@@ -828,14 +826,12 @@ class PCIIface(PCIIfaceBase):
       'simd_arrays_per_engine': max_sh_per_se, 'lds_size_in_kb': self.dev_impl.gc_info.gc_lds_size, 'num_xcc': self.dev_impl.gfx.xccs,
       'gfx_target_version': {90403: 90402}.get(gfxver, gfxver)}
 
-  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0,
-                   xcc_id=0, idx=0):
+  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0, xcc_id=0):
     assert cwsr_buffer is None, "no cwsr buffer for am"
 
     if queue_type == kfd.KFD_IOC_QUEUE_TYPE_SDMA:
-      assert idx <= 3, "only 4 SDMA queues supported in am"
       pv = self.dev_impl.sdma.setup_ring(ring_addr=ring.va_addr, ring_size=ring.size, rptr_addr=gart.va_addr+rptr, wptr_addr=gart.va_addr+wptr,
-            doorbell=(doorbell_index:=am.AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0 + idx * 0xA * 4), pipe=0, queue=idx)
+                                    doorbell=(doorbell_index:=am.AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0), pipe=0, queue=0)
     else:
       pv = self.dev_impl.gfx.setup_ring(ring_addr=ring.va_addr, ring_size=ring.size, rptr_addr=gart.va_addr+rptr, wptr_addr=gart.va_addr+wptr,
         eop_addr=eop_buffer.va_addr, eop_size=eop_buffer.size, doorbell=(doorbell_index:=am.AMDGPU_NAVI10_DOORBELL_MEC_RING0), pipe=0,
@@ -879,10 +875,9 @@ class USBIface(PCIIface):
     barview = self.pci_dev.map_bar(bar=0, off=mapping.paddrs[0][0], size=mapping.size) if cpu_access else None
     return HCQBuffer(mapping.va_addr, size, meta=PCIAllocationMeta(mapping, has_cpu_mapping=False), view=barview, owner=self.dev)
 
-  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0,
-                   xcc_id=0, idx=0):
+  def create_queue(self, queue_type, ring, gart, rptr, wptr, eop_buffer=None, cwsr_buffer=None, ctl_stack_size=0, ctx_save_restore_size=0, xcc_id=0):
     if queue_type == kfd.KFD_IOC_QUEUE_TYPE_COMPUTE: self.pci_dev.usb._pci_cacheable += [(ring.cpu_view().addr, ring.size)]
-    return super().create_queue(queue_type, ring, gart, rptr, wptr, eop_buffer, cwsr_buffer, ctl_stack_size, ctx_save_restore_size, xcc_id, idx)
+    return super().create_queue(queue_type, ring, gart, rptr, wptr, eop_buffer, cwsr_buffer, ctl_stack_size, ctx_save_restore_size, xcc_id)
 
   def sleep(self, timeout): pass
 
@@ -936,7 +931,8 @@ class AMDDevice(HCQCompiled):
       0x2000 if self.is_usb() else (16 << 20), eop_buffer_size=0x1000,
       ctx_save_restore_size=0 if self.is_am() else wg_data_size + ctl_stack_size, ctl_stack_size=ctl_stack_size, debug_memory_size=debug_memory_size)
 
-    self.max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000
+    max_copy_size = 0x40000000 if self.iface.ip_versions[am.SDMA0_HWIP][0] >= 5 else 0x400000
+    self.sdma_queue = self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20))
 
     compilers = CompilerSet([CompilerPair(functools.partial(AMDHIPRenderer, self.arch), None),
                              CompilerPair(functools.partial(AMDLLVMRenderer, self.arch), None, AMD_LLVM),
@@ -944,7 +940,7 @@ class AMDDevice(HCQCompiled):
 
     super().__init__(device, AMDAllocator(self), compilers, functools.partial(AMDProgram, self), AMDSignal,
                      functools.partial(AMDComputeAQLQueue if self.is_aql else AMDComputeQueue, self),
-                     functools.partial(AMDCopyQueue, self, max_copy_size=self.max_copy_size),
+                     functools.partial(AMDCopyQueue, self, max_copy_size=max_copy_size),
                      kernargs_size=(8 << 10) if self.is_usb() else (16 << 20), sigalloc_size=0x100 if self.is_usb() else 0x1000)
 
     # Scratch setup
@@ -980,7 +976,7 @@ class AMDDevice(HCQCompiled):
       self.sqtt_wptrs = self.allocator.alloc(round_up(self.se_cnt * 4, 0x1000), BufferSpec(cpu_access=True, nolru=True))
       self.sqtt_next_cmd_id = itertools.count(0)
 
-  def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0, idx=0):
+  def create_queue(self, queue_type, ring_size, ctx_save_restore_size=0, eop_buffer_size=0, ctl_stack_size=0, debug_memory_size=0):
     ring = self.iface.alloc(ring_size, uncached=True, cpu_access=True)
     gart = self.iface.alloc(0x100, uncached=True, cpu_access=True)
 
@@ -997,10 +993,7 @@ class AMDDevice(HCQCompiled):
 
     return (self.iface.create_queue(queue_type, ring, gart, rptr=getattr(hsa.amd_queue_t, 'read_dispatch_id').offset,
             wptr=getattr(hsa.amd_queue_t, 'write_dispatch_id').offset, eop_buffer=eop_buffer, cwsr_buffer=cwsr_buffer,
-            ctx_save_restore_size=ctx_save_restore_size, ctl_stack_size=ctl_stack_size, idx=idx))
-
-  @functools.lru_cache(None)
-  def sdma_queue(self, idx:int=0): return self.create_queue(kfd.KFD_IOC_QUEUE_TYPE_SDMA, 0x200 if self.is_usb() else (16 << 20), idx=idx)
+            ctx_save_restore_size=ctx_save_restore_size, ctl_stack_size=ctl_stack_size))
 
   def _ensure_has_local_memory(self, private_segment_size):
     if self.max_private_segment_size >= private_segment_size: return

tinygrad/runtime/ops_nv.py

@@ -165,10 +165,6 @@ class NVComputeQueue(NVCommandQueue):
   def _submit(self, dev:NVDevice): self._submit_to_gpfifo(dev, dev.compute_gpfifo)
 
 class NVCopyQueue(NVCommandQueue):
-  def __init__(self, queue_idx=0):
-    self.queue_idx = queue_idx
-    super().__init__()
-
   def copy(self, dest:sint, src:sint, copy_size:int):
     for off in range(0, copy_size, step:=(1 << 31)):
       self.nvm(4, nv_gpu.NVC6B5_OFFSET_IN_UPPER, *data64(src+off), *data64(dest+off))

tinygrad/runtime/ops_qcom.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import os, ctypes, functools, mmap, struct, array, math, sys, weakref, contextlib
 assert sys.platform != 'win32'
+from types import SimpleNamespace
 from typing import Any, cast
 from tinygrad.device import BufferSpec, CompilerSet, CompilerPair
 from tinygrad.runtime.support.hcq import HCQBuffer, HWQueue, HCQProgram, HCQCompiled, HCQAllocatorBase, HCQSignal, HCQArgsState, BumpAllocator
@@ -10,7 +11,7 @@ from tinygrad.runtime.ops_cl import CLCompiler, CLDevice
 from tinygrad.renderer.cstyle import QCOMRenderer
 from tinygrad.renderer.nir import IR3Renderer
 from tinygrad.helpers import getenv, mv_address, to_mv, round_up, data64_le, prod, fromimport, cpu_profile, lo32, PROFILE, suppress_finalizing
-from tinygrad.helpers import next_power2, flatten, QCOM_IR3, QCOM_CC
+from tinygrad.helpers import flatten, QCOM_IR3, QCOM_CC
 from tinygrad.runtime.support.system import System
 if getenv("IOCTL"): import extra.qcom_gpu_driver.opencl_ioctl  # noqa: F401  # pylint: disable=unused-import
 
@@ -25,7 +26,7 @@ def _qreg_exec(__reg, __val=0, **kwargs):
   return __val
 qreg: Any = type("QREG", (object,), {name[4:].lower(): functools.partial(_qreg_exec, name) for name in mesa.__dict__.keys() if name[:4] == 'REG_'})
 
-def ctz(v): return (v & -v).bit_length() - 1
+def next_power2(x): return 1 if x == 0 else 1 << (x - 1).bit_length()
 
 def parity(val: int):
   for i in range(4,1,-1): val ^= val >> (1 << i)
@@ -190,29 +191,37 @@ class QCOMComputeQueue(HWQueue):
 class QCOMArgsState(HCQArgsState):
   def __init__(self, buf:HCQBuffer, prg:QCOMProgram, bufs:tuple[HCQBuffer, ...], vals:tuple[int, ...]=()):
     super().__init__(buf, prg, bufs, vals=vals)
-    ctypes.memset(cast(int, self.buf.va_addr), 0, prg.kernargs_alloc_size)
 
-    ubos, uavs = [b for b in bufs if b.image is None], [b for b in bufs if b.image is not None]
-    ibos, texs = (uavs, []) if prg.tex_cnt == 0 else (uavs[:-prg.tex_cnt], uavs[-prg.tex_cnt:])
+    if len(bufs) + len(vals) != len(prg.buf_info): raise RuntimeError(f'incorrect args size given={len(bufs)+len(vals)} != want={len(prg.buf_info)}')
+
+    self.buf_info, self.args_info = prg.buf_info[:len(bufs)], prg.buf_info[len(bufs):]
+
+    ctypes.memset(cast(int, self.buf.va_addr), 0, prg.kernargs_alloc_size)
     for cnst_val,cnst_off,cnst_sz in prg.consts_info: to_mv(self.buf.va_addr + cnst_off, cnst_sz)[:] = cnst_val.to_bytes(cnst_sz, byteorder='little')
 
     if prg.samp_cnt > 0: to_mv(self.buf.va_addr + prg.samp_off, len(prg.samplers) * 4).cast('I')[:] = array.array('I', prg.samplers)
-    if prg.NIR:
-      self.bind_sints_to_buf(*[b.va_addr for b in ubos], buf=self.buf, fmt='Q', offset=prg.buf_off)
-      self.bind_sints_to_buf(*vals, buf=self.buf, fmt='I', offset=prg.buf_off + len(ubos) * 8)
-    else:
-      for i, b in enumerate(ubos): self.bind_sints_to_buf(b.va_addr, buf=self.buf, fmt='Q', offset=prg.buf_offs[i])
-      for i, v in enumerate(vals): self.bind_sints_to_buf(v, buf=self.buf, fmt='I', offset=prg.buf_offs[i+len(ubos)])
+    for i, b in enumerate(bufs):
+      if prg.buf_info[i].type in {BUFTYPE_TEX, BUFTYPE_IBO}:
+        obj = b.texture_info.desc if prg.buf_info[i].type is BUFTYPE_TEX else b.texture_info.ibo
+        to_mv(self.buf.va_addr + prg.buf_info[i].offset, len(obj) * 4).cast('I')[:] = array.array('I', obj)
+      self.bind_sints_to_buf(b.va_addr, buf=self.buf, fmt='Q', offset=self.buf_info[i].offset+(0 if self.buf_info[i].type is BUFTYPE_BUF else 16))
 
-    def _tex(b, ibo=False):
-      fmt = mesa.FMT6_32_32_32_32_FLOAT if b.image.itemsize == 4 else mesa.FMT6_16_16_16_16_FLOAT
-      return [qreg.a6xx_tex_const_0(fmt=fmt) if ibo else qreg.a6xx_tex_const_0(0x8, swiz_x=0, swiz_y=1, swiz_z=2, swiz_w=3, fmt=fmt),
-              qreg.a6xx_tex_const_1(width=b.image.shape[1], height=b.image.shape[0]),
-              qreg.a6xx_tex_const_2(type=mesa.A6XX_TEX_2D, pitch=b.image.pitch, pitchalign=ctz(b.image.pitch)-6), 0, *data64_le(b.va_addr),
-              qreg.a6xx_tex_const_6(plane_pitch=0x400000), qreg.a6xx_tex_const_7(13), 0, 0, 0, 0, 0, 0, 0, 0]
+    for i, v in enumerate(vals): self.bind_sints_to_buf(v, buf=self.buf, fmt='I', offset=self.args_info[i].offset)
 
-    self.bind_sints_to_buf(*flatten(map(_tex, texs)), buf=self.buf, fmt='I', offset=prg.tex_off)
-    self.bind_sints_to_buf(*flatten(map(functools.partial(_tex, ibo=True), ibos)), buf=self.buf, fmt='I', offset=prg.ibo_off)
+class IR3ArgsState(HCQArgsState):
+  def __init__(self, buf:HCQBuffer, prg:QCOMProgram, bufs:tuple[HCQBuffer, ...], vals:tuple[int, ...]=()):
+    super().__init__(buf, prg, bufs, vals=vals)
+    ctypes.memset(cast(int, self.buf.va_addr), 0, prg.kernargs_alloc_size)
+    to_mv(self.buf.va_addr + prg.imm_off, len(prg.imm_vals))[:] = prg.imm_vals
+
+    ubos, uavs = [b for b in bufs if b.texture_info is None], [b for b in bufs if b.texture_info is not None]
+    ibos, texs = (uavs, []) if prg.tex_cnt == 0 else (uavs[:-prg.tex_cnt], uavs[-prg.tex_cnt:]) # textures are at the end
+
+    if prg.samp_cnt > 0: to_mv(self.buf.va_addr + prg.samp_off, len(prg.samplers) * 4).cast('I')[:] = array.array('I', prg.samplers)
+    self.bind_sints_to_buf(*[b.va_addr for b in ubos], buf=self.buf, fmt='Q', offset=prg.buf_off)
+    self.bind_sints_to_buf(*vals, buf=self.buf, fmt='I', offset=prg.buf_off + len(ubos) * 8)
+    self.bind_sints_to_buf(*flatten([b.texture_info.desc + ([0] * 8) for b in texs]), buf=self.buf, fmt='I', offset=prg.tex_off)
+    self.bind_sints_to_buf(*flatten([b.texture_info.ibo + ([0] * 8) for b in ibos]), buf=self.buf, fmt='I', offset=prg.ibo_off)
 
 class QCOMProgram(HCQProgram):
   def __init__(self, dev: QCOMDevice, name: str, lib: bytes):
@@ -237,11 +246,10 @@ class QCOMProgram(HCQProgram):
 
       self.tex_off, self.ibo_off, self.samp_off = 2048, 2048 + 0x40 * self.tex_cnt, 2048 + 0x40 * (self.tex_cnt + self.ibo_cnt)
       self.fregs, self.hregs = v.info.max_reg + 1, v.info.max_half_reg + 1
-      self.consts_info:list[tuple] = []
     else: self._parse_lib()
 
     self.lib_gpu: HCQBuffer = self.dev.allocator.alloc(self.image_size, buf_spec:=BufferSpec(cpu_access=True, nolru=True))
-    to_mv(self.lib_gpu.va_addr, self.image_size)[:] = self.image
+    to_mv(cast(int, self.lib_gpu.va_addr), self.image_size)[:] = self.image
 
     self.pvtmem_size_per_item: int = round_up(self.pvtmem, 512) >> 9
     self.pvtmem_size_total: int = self.pvtmem_size_per_item * 128 * 2
@@ -251,7 +259,7 @@ class QCOMProgram(HCQProgram):
     dev._ensure_stack_size(self.hw_stack_offset * 4)
 
     kernargs_alloc_size = round_up(2048 + (self.tex_cnt + self.ibo_cnt) * 0x40 + len(self.samplers) * 4, 0x100)
-    super().__init__(QCOMArgsState, self.dev, self.name, kernargs_alloc_size=kernargs_alloc_size)
+    super().__init__(IR3ArgsState if self.NIR else QCOMArgsState, self.dev, self.name, kernargs_alloc_size=kernargs_alloc_size)
     weakref.finalize(self, self._fini, self.dev, self.lib_gpu, buf_spec)
 
   def __call__(self, *bufs, global_size:tuple[int,int,int]=(1,1,1), local_size:tuple[int,int,int]=(1,1,1), vals:tuple[int, ...]=(), wait=False):
@@ -271,7 +279,7 @@ class QCOMProgram(HCQProgram):
     self.pvtmem, self.shmem = _read_lib(self.lib, image_desc_off+0xc8), _read_lib(self.lib, image_desc_off+0xd8)
 
     # Fill up constants and buffers info
-    self.consts_info = []
+    self.buf_info, self.consts_info = [], []
 
     # Collect sampler info.
     self.samp_cnt = samp_cnt_in_file = _read_lib(self.lib, image_desc_off + 0xdc)
@@ -283,17 +291,20 @@ class QCOMProgram(HCQProgram):
     else: self.samplers = []
 
     # Collect kernel arguments (buffers) info.
-    bdoff, binfos = round_up(image_desc_off + 0x158 + len(self.name), 4) + 8 * samp_cnt_in_file, []
+    bdoff = round_up(image_desc_off + 0x158 + len(self.name), 4) + 8 * samp_cnt_in_file
     while bdoff + 32 <= len(self.lib):
-      length, _, _, offset_words, _, _, _, typ = struct.unpack("8I", self.lib[bdoff:bdoff+32])
+      length, _, _, offset_words, _, _, _, typ = struct.unpack("IIIIIIII", self.lib[bdoff:bdoff+32])
       if length == 0: break
-      binfos.append((offset_words * 4, typ))
+      self.buf_info.append(SimpleNamespace(offset=offset_words * 4, type=typ))
       bdoff += length
-    self.buf_offs = [off for off,typ in binfos if typ not in {BUFTYPE_TEX, BUFTYPE_IBO}]
 
     # Setting correct offsets to textures/ibos.
-    self.tex_cnt, self.ibo_cnt = sum(typ is BUFTYPE_TEX for _,typ in binfos), sum(typ is BUFTYPE_IBO for _,typ in binfos)
+    self.tex_cnt, self.ibo_cnt = sum(x.type is BUFTYPE_TEX for x in self.buf_info), sum(x.type is BUFTYPE_IBO for x in self.buf_info)
     self.ibo_off, self.tex_off, self.samp_off = 2048, 2048 + 0x40 * self.ibo_cnt, 2048 + 0x40 * self.tex_cnt + 0x40 * self.ibo_cnt
+    cur_ibo_off, cur_tex_off = self.ibo_off, self.tex_off
+    for x in self.buf_info:
+      if x.type is BUFTYPE_IBO: x.offset, cur_ibo_off = cur_ibo_off, cur_ibo_off + 0x40
+      elif x.type is BUFTYPE_TEX: x.offset, cur_tex_off = cur_tex_off, cur_tex_off + 0x40
 
     if _read_lib(self.lib, 0xb0) != 0: # check if we have constants.
       cdoff = _read_lib(self.lib, 0xac)
@@ -311,10 +322,28 @@ class QCOMTextureInfo:
     self.pitch, self.real_stride, self.desc, self.ibo = pitch, real_stride, desc, ibo
 
 class QCOMAllocator(HCQAllocatorBase):
-  def _alloc(self, size:int, opts:BufferSpec) -> HCQBuffer:
+  def _alloc(self, size:int, options:BufferSpec) -> HCQBuffer:
     # Recalculate real size for texture
-    if opts.image is not None: size = opts.image.pitch* opts.image.shape[0]
-    return self.dev._gpu_map(opts.external_ptr, size, image=opts.image) if opts.external_ptr else self.dev._gpu_alloc(size, image=opts.image)
+    if options.image is not None:
+      imgw, imgh, itemsize_log = options.image.shape[1], options.image.shape[0], int(math.log2(options.image.itemsize))
+      pitchalign = max(6, 11 - int(math.log2(imgh))) if imgh > 1 else 6
+      align_up = max(1, (8 // itemsize_log + 1) - imgh // 32) if pitchalign == 6 else (2 ** (pitchalign - itemsize_log - 2))
+
+      granularity = 128 if options.image.itemsize == 4 else 256
+      pitch_add = (1 << pitchalign) if min(next_power2(imgw), round_up(imgw, granularity)) - align_up + 1 <= imgw and imgw > granularity//2 else 0
+      pitch = round_up((real_stride:=imgw * 4 * options.image.itemsize), 1 << pitchalign) + pitch_add
+      size = pitch * imgh
+
+    buf = self.dev._gpu_map(options.external_ptr, size) if options.external_ptr else self.dev._gpu_alloc(size)
+
+    if options.image is not None:
+      tex_fmt = mesa.FMT6_32_32_32_32_FLOAT if options.image.itemsize == 4 else mesa.FMT6_16_16_16_16_FLOAT
+      desc = [qreg.a6xx_tex_const_0(0x8, swiz_x=0, swiz_y=1, swiz_z=2, swiz_w=3, fmt=tex_fmt), qreg.a6xx_tex_const_1(width=imgw, height=imgh),
+              qreg.a6xx_tex_const_2(type=mesa.A6XX_TEX_2D, pitch=pitch, pitchalign=pitchalign-6), 0,
+              *data64_le(buf.va_addr), qreg.a6xx_tex_const_6(plane_pitch=0x400000), qreg.a6xx_tex_const_7(13)]
+
+      buf.texture_info = QCOMTextureInfo(pitch, real_stride, desc, [desc[0] & (~0xffff), *desc[1:len(desc)]])
+    return buf
 
   def _do_copy(self, src_addr, dest_addr, src_size, real_size, src_stride, dest_stride, prof_text, dest_off=0, src_off=0):
     with cpu_profile(prof_text, self.dev.device, is_copy=True):
@@ -323,13 +352,13 @@ class QCOMAllocator(HCQAllocatorBase):
         src_off, dest_off = src_off+src_stride, dest_off+dest_stride
 
   def _copyin(self, dest:HCQBuffer, src:memoryview):
-    stride, pitch = (dest.image.shape[1] * 4 * dest.image.itemsize, dest.image.pitch) if dest.image else (src.nbytes, src.nbytes)
+    stride, pitch = (src.nbytes, src.nbytes) if (ti:=cast(QCOMTextureInfo, dest.texture_info)) is None else (ti.real_stride, ti.pitch)
     self._do_copy(mv_address(src), dest.cpu_view().addr, src.nbytes, stride, stride, pitch, f"TINY -> {self.dev.device}")
 
   def _copyout(self, dest:memoryview, src:HCQBuffer):
     self.dev.synchronize()
 
-    stride, pitch = (src.image.shape[1] * 4 * src.image.itemsize, src.image.pitch) if src.image else (src.size, src.size)
+    stride, pitch = (src.size, src.size) if (ti:=cast(QCOMTextureInfo, src.texture_info)) is None else (ti.real_stride, ti.pitch)
     self._do_copy(src.cpu_view().addr, mv_address(dest), src.size, stride, pitch, stride, f"{self.dev.device} -> TINY")
 
   def _as_buffer(self, src:HCQBuffer) -> memoryview:
@@ -376,7 +405,7 @@ class QCOMDevice(HCQCompiled):
     super().__init__(device, QCOMAllocator(self), compilers, functools.partial(QCOMProgram, self), QCOMSignal,
                      functools.partial(QCOMComputeQueue, self), None)
 
-  def _gpu_alloc(self, size:int, flags:int=0, uncached=False, fill_zeroes=False, **kwargs) -> HCQBuffer:
+  def _gpu_alloc(self, size:int, flags:int=0, uncached=False, fill_zeroes=False) -> HCQBuffer:
     flags |= flag("KGSL_MEMALIGN", alignment_hint:=12) | kgsl.KGSL_MEMFLAGS_USE_CPU_MAP
     if uncached: flags |= flag("KGSL_CACHEMODE", kgsl.KGSL_CACHEMODE_UNCACHED)
 
@@ -384,15 +413,15 @@ class QCOMDevice(HCQCompiled):
     va_addr = self.fd.mmap(0, bosz, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED, alloc.id * 0x1000)
 
     if fill_zeroes: ctypes.memset(va_addr, 0, size)
-    return HCQBuffer(va_addr=va_addr, size=size, meta=(alloc, True), view=MMIOInterface(va_addr, size, fmt='B'), owner=self, **kwargs)
+    return HCQBuffer(va_addr=va_addr, size=size, meta=(alloc, True), view=MMIOInterface(va_addr, size, fmt='B'), owner=self)
 
-  def _gpu_map(self, ptr:int, size:int, **kwargs) -> HCQBuffer:
+  def _gpu_map(self, ptr:int, size:int) -> HCQBuffer:
     ptr_aligned, size_aligned = (ptr & ~0xfff), round_up(size + (ptr & 0xfff), 0x1000)
     try:
-      mi = kgsl.IOCTL_KGSL_MAP_USER_MEM(self.fd, hostptr=ptr_aligned, len=size_aligned, memtype=kgsl.KGSL_USER_MEM_TYPE_ADDR)
-      return HCQBuffer(mi.gpuaddr + (ptr - ptr_aligned), size=size, meta=(mi, False), view=MMIOInterface(ptr, size, fmt='B'), owner=self, **kwargs)
+      mapinfo = kgsl.IOCTL_KGSL_MAP_USER_MEM(self.fd, hostptr=ptr_aligned, len=size_aligned, memtype=kgsl.KGSL_USER_MEM_TYPE_ADDR)
+      return HCQBuffer(mapinfo.gpuaddr + (ptr - ptr_aligned), size=size, meta=(mapinfo, False), view=MMIOInterface(ptr, size, fmt='B'), owner=self)
     except OSError as e:
-      if e.errno == 14: return HCQBuffer(va_addr=ptr, size=size, meta=(None, False), view=MMIOInterface(ptr, size, fmt='B'), owner=self, **kwargs)
+      if e.errno == 14: return HCQBuffer(va_addr=ptr, size=size, meta=(None, False), view=MMIOInterface(ptr, size, fmt='B'), owner=self)
       raise RuntimeError("Failed to map external pointer to GPU memory") from e
 
   def _gpu_free(self, mem:HCQBuffer):

tinygrad/runtime/support/am/ip.py

@@ -189,18 +189,16 @@ class AM_SMU(AM_IP):
     return table_t.from_buffer(bytearray(self.adev.vram.view(self.driver_table_paddr, ctypes.sizeof(table_t))[:]))
 
   def set_clocks(self, level):
-    if not hasattr(self, 'clcks'):
-      clks = [self.smu_mod.PPCLK_UCLK, self.smu_mod.PPCLK_FCLK, self.smu_mod.PPCLK_SOCCLK]
-      if self.adev.ip_ver[am.MP0_HWIP] not in {(13,0,6), (13,0,12)}: clks.append(self.smu_mod.PPCLK_GFXCLK)
+    if self.adev.ip_ver[am.MP0_HWIP] in {(13,0,6), (13,0,12)}: return # TODO
 
+    if not hasattr(self, 'clcks'):
       self.clcks = {}
-      for clck in clks:
+      for clck in [self.smu_mod.PPCLK_GFXCLK, self.smu_mod.PPCLK_UCLK, self.smu_mod.PPCLK_FCLK, self.smu_mod.PPCLK_SOCCLK]:
         cnt = self._send_msg(self.smu_mod.PPSMC_MSG_GetDpmFreqByIndex, (clck<<16)|0xff, read_back_arg=True)&0x7fffffff
         self.clcks[clck] = [self._send_msg(self.smu_mod.PPSMC_MSG_GetDpmFreqByIndex, (clck<<16)|i, read_back_arg=True)&0x7fffffff for i in range(cnt)]
 
     for clck, vals in self.clcks.items():
-      if not vals: continue
-      with contextlib.suppress(TimeoutError): self._send_msg(self.smu_mod.PPSMC_MSG_SetSoftMinByFreq, clck << 16 | (vals[level]), timeout=20)
+      self._send_msg(self.smu_mod.PPSMC_MSG_SetSoftMinByFreq, clck << 16 | (vals[level]))
       self._send_msg(self.smu_mod.PPSMC_MSG_SetSoftMaxByFreq, clck << 16 | (vals[level]))
 
   def _smu_cmn_send_msg(self, msg:int, param=0, debug=False):
@@ -414,34 +412,33 @@ class AM_IH(AM_IP):
     self.adev.regIH_RB_RPTR.write(wptr % self.ring_size)
 
 class AM_SDMA(AM_IP):
-  def init_sw(self): self.sdma_reginst, self.sdma_name = [], "F32" if self.adev.ip_ver[am.SDMA0_HWIP] < (7,0,0) else "MCU"
+  def init_sw(self): self.sdma_name = "F32" if self.adev.ip_ver[am.SDMA0_HWIP] < (7,0,0) else "MCU"
   def init_hw(self):
     for pipe_id in range(1):
-      pipe, inst = ("", pipe_id) if self.adev.ip_ver[am.SDMA0_HWIP] < (5,0,0) else (str(pipe_id), 0)
+      pipe = "" if self.adev.ip_ver[am.SDMA0_HWIP] < (5,0,0) else str(pipe_id)
 
       if self.adev.ip_ver[am.SDMA0_HWIP] >= (6,0,0):
-        self.adev.reg(f"regSDMA{pipe}_WATCHDOG_CNTL").update(queue_hang_count=100, inst=inst) # 10s, 100ms per unit
-        self.adev.reg(f"regSDMA{pipe}_UTCL1_CNTL").update(resp_mode=3, redo_delay=9, inst=inst)
+        self.adev.reg(f"regSDMA{pipe}_WATCHDOG_CNTL").update(queue_hang_count=100) # 10s, 100ms per unit
+        self.adev.reg(f"regSDMA{pipe}_UTCL1_CNTL").update(resp_mode=3, redo_delay=9)
 
         # rd=noa, wr=bypass
-        self.adev.reg(f"regSDMA{pipe}_UTCL1_PAGE").update(rd_l2_policy=2, wr_l2_policy=3, **({'llc_noalloc':1} if self.sdma_name == "F32" else {}),
-                                                          inst=inst)
-        self.adev.reg(f"regSDMA{pipe}_{self.sdma_name}_CNTL").update(halt=0, **{f"{'th1_' if self.sdma_name == 'F32' else ''}reset":0}, inst=inst)
+        self.adev.reg(f"regSDMA{pipe}_UTCL1_PAGE").update(rd_l2_policy=2, wr_l2_policy=3, **({'llc_noalloc':1} if self.sdma_name == "F32" else {}))
+        self.adev.reg(f"regSDMA{pipe}_{self.sdma_name}_CNTL").update(halt=0, **{f"{'th1_' if self.sdma_name == 'F32' else ''}reset":0})
 
       self.adev.reg(f"regSDMA{pipe}_CNTL").update(ctxempty_int_enable=1, trap_enable=1,
-        **({'utc_l1_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP] <= (5,2,0) else {}), inst=inst)
+        **({'utc_l1_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP] <= (5,2,0) else {}))
 
     if self.adev.ip_ver[am.NBIO_HWIP] in {(7,9,0), (7,9,1)}:
-      for i in range(16): self.adev.reg(f"regDOORBELL0_CTRL_ENTRY_{i+1}").write(**{f"bif_doorbell{i+1}_range_size_entry":4,
-                                        f"bif_doorbell{i+1}_range_offset_entry":(am.AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0 + i * 0xA) * 2})
+      self.adev.regDOORBELL0_CTRL_ENTRY_1.write(bif_doorbell1_range_offset_entry=am.AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0*2,
+        bif_doorbell1_range_size_entry=4)
       self.adev.soc.doorbell_enable(port=2, awid=0xe, awaddr_31_28_value=0x1, offset=0xe, size=4)
     else: self.adev.soc.doorbell_enable(port=2, awid=0xe, awaddr_31_28_value=0x3, offset=am.AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0*2, size=4)
 
   def fini_hw(self):
-    for reg, inst in self.sdma_reginst:
-      self.adev.reg(f"{reg}_RB_CNTL").update(rb_enable=0, inst=inst)
-      self.adev.reg(f"{reg}_IB_CNTL").update(ib_enable=0, inst=inst)
+    reg, inst = ("regSDMA_GFX", 0) if self.adev.ip_ver[am.SDMA0_HWIP][:2] == (4,4) else ("regSDMA0_QUEUE0", 0)
 
+    self.adev.reg(f"{reg}_RB_CNTL").update(rb_enable=0, inst=inst)
+    self.adev.reg(f"{reg}_IB_CNTL").update(ib_enable=0, inst=inst)
     if self.adev.ip_ver[am.SDMA0_HWIP] >= (6,0,0):
       self.adev.regGRBM_SOFT_RESET.write(soft_reset_sdma0=1)
       time.sleep(0.01)
@@ -449,8 +446,7 @@ class AM_SDMA(AM_IP):
 
   def setup_ring(self, ring_addr:int, ring_size:int, rptr_addr:int, wptr_addr:int, doorbell:int, pipe:int, queue:int) -> int:
     # Setup the ring
-    reg, inst = ("regSDMA_GFX", pipe+queue*4) if self.adev.ip_ver[am.SDMA0_HWIP][:2] == (4,4) else (f"regSDMA{pipe}_QUEUE{queue}", 0)
-    self.sdma_reginst.append((reg, inst))
+    reg, inst = ("regSDMA_GFX", pipe*4+queue) if self.adev.ip_ver[am.SDMA0_HWIP][:2] == (4,4) else (f"regSDMA{pipe}_QUEUE{queue}", 0)
 
     self.adev.reg(f"{reg}_MINOR_PTR_UPDATE").write(0x1, inst=inst)
     if not self.adev.partial_boot: self.adev.wreg_pair(f"{reg}_RB_RPTR", "", "_HI", 0, inst=inst)
@@ -464,7 +460,7 @@ class AM_SDMA(AM_IP):
     self.adev.reg(f"{reg}_RB_CNTL").write(**({f'{self.sdma_name.lower()}_wptr_poll_enable':1} if self.adev.ip_ver[am.SDMA0_HWIP][:2]!=(4,4) else {}),
       rb_vmid=0, rptr_writeback_enable=1, rptr_writeback_timer=4, rb_enable=1, rb_priv=1, rb_size=(ring_size//4).bit_length()-1, inst=inst)
     self.adev.reg(f"{reg}_IB_CNTL").update(ib_enable=1, inst=inst)
-    return self.adev.reg(f"{reg}_RB_WPTR").read(inst=inst) | (self.adev.reg(f"{reg}_RB_WPTR_HI").read(inst=inst) << 32)
+    return self.adev.reg(f"{reg}_RB_WPTR").read() | (self.adev.reg(f"{reg}_RB_WPTR_HI").read() << 32)
 
 class AM_PSP(AM_IP):
   def init_sw(self):

tinygrad/runtime/support/hcq.py

@@ -8,7 +8,6 @@ from tinygrad.device import BufferSpec, Compiled, LRUAllocator, ProfileDeviceEve
 from tinygrad.uop.ops import sym_infer, sint, UOp
 from tinygrad.runtime.autogen import libc
 from tinygrad.runtime.support.memory import BumpAllocator
-from tinygrad.dtype import ImageDType
 
 class MMIOInterface:
   def __init__(self, addr:int, nbytes:int, fmt='B'): self.mv, self.addr, self.nbytes, self.fmt = to_mv(addr, nbytes).cast(fmt), addr, nbytes, fmt
@@ -355,7 +354,7 @@ class HCQCompiled(Compiled, Generic[SignalType]):
   cpu_devices: list[HCQCompiled] = []
 
   def __init__(self, device:str, allocator:HCQAllocatorBase, compilers:CompilerSet, runtime, signal_t:Type[SignalType],
-               comp_queue_t:Callable[..., HWQueue], copy_queue_t:Callable[..., HWQueue]|None=None, kernargs_size=(16 << 20), sigalloc_size=0x1000):
+               comp_queue_t:Callable[[], HWQueue], copy_queue_t:Callable[[], HWQueue]|None=None, kernargs_size=(16 << 20), sigalloc_size=0x1000):
     self.device_id:int = int(device.split(":")[1]) if ":" in device else 0
 
     from tinygrad.runtime.graph.hcq import HCQGraph
@@ -456,14 +455,14 @@ class HCQCompiled(Compiled, Generic[SignalType]):
     if hasattr(self, 'iface') and hasattr(self.iface, 'device_fini'): self.iface.device_fini()
 
 class HCQBuffer:
-  def __init__(self, va_addr:sint, size:int, image:ImageDType|None=None, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None,
+  def __init__(self, va_addr:sint, size:int, texture_info:Any=None, meta:Any=None, _base:HCQBuffer|None=None, view:MMIOInterface|None=None,
                owner:HCQCompiled|None=None):
-    self.va_addr, self.size, self.image, self.meta, self._base, self.view = va_addr, size, image, meta, _base, view
+    self.va_addr, self.size, self.texture_info, self.meta, self._base, self.view = va_addr, size, texture_info, meta, _base, view
     self._devs, self.owner = ([owner] if owner is not None else []), owner
     self._mappings:dict[HCQCompiled, HCQBuffer] = {} # mapping to the other devices
 
   def offset(self, offset:int=0, size:int|None=None) -> HCQBuffer:
-    return HCQBuffer(self.va_addr+offset, size or (self.size - offset), owner=self.owner, image=self.image, meta=self.meta,
+    return HCQBuffer(self.va_addr+offset, size or (self.size - offset), owner=self.owner, texture_info=self.texture_info, meta=self.meta,
       _base=self._base or self, view=(self.view.view(offset=offset, size=size) if self.view is not None else None))
 
   def cpu_view(self) -> MMIOInterface:

tinygrad/schedule/multi.py

@@ -1,6 +1,6 @@
 from typing import cast
 import functools, itertools, operator
-from tinygrad.helpers import all_same, all_int, prod, DEBUG, RING, ALL2ALL, getenv
+from tinygrad.helpers import all_same, all_int, prod, DEBUG, RING, getenv
 from tinygrad.uop.ops import Ops, UOp, sint, PatternMatcher, UPat, GroupOp, graph_rewrite_map, graph_rewrite
 from tinygrad.device import Device
 
@@ -35,49 +35,45 @@ def handle_allreduce(buf:UOp, red:UOp) -> UOp|None:
   if not isinstance(buf.device, tuple): return None
   assert all_int(buf.shape), f"does not support symbolic shape {buf.shape}"
   n_lbs, shape, numel = len(buf.device), buf.shape, prod(buf.shape)
-
   # ring allreduce doesn't provide a benefit with only 2 nodes or where number of elements is less than 256k (empirically)
   # fallback to naive allreduce to save on kernel dispatch, chunking and reassembling chunks.
-  use_all2all = (ALL2ALL >= 2 or (n_lbs > 2 and numel > getenv("RING_ALLREDUCE_THRESHOLD", 256_000) and ALL2ALL >= 1))
-  use_ring = not use_all2all and (RING >= 2 or (n_lbs > 2 and numel > getenv("RING_ALLREDUCE_THRESHOLD", 256_000) and RING >= 1))
-  if DEBUG >= 2: print(f"{'ALL2ALL' if use_all2all else 'RING' if use_ring else 'NAIVE'} ALLREDUCE {n_lbs}x{numel} | {buf.dtype}")
+  use_ring = (RING >= 2 or (n_lbs > 2 and numel > getenv("RING_ALLREDUCE_THRESHOLD", 256_000) and RING >= 1))
+  if DEBUG >= 2: print(f"{'RING ALLREDUCE' if use_ring else 'NAIVE ALLREDUCE'} {n_lbs}x{numel} | {buf.dtype}")
 
   # contiguous before we copy it
   buf = buf.contiguous()
 
-  # naive: copy to all devices. if you shrink later, that'll be handled
-  if not use_ring and not use_all2all:
-    return functools.reduce(lambda x,y: x.alu(red.arg, y), [UOp(Ops.COPY, buf.dtype, (buf.mselect(i), red.src[1])) for i in range(n_lbs)])
+  # copy to all devices. if you shrink later, that'll be handled
+  if not use_ring: return functools.reduce(lambda x,y: x.alu(red.arg, y),
+                                           [UOp(Ops.COPY, buf.dtype, (buf.mselect(i), red.src[1])) for i in range(len(buf.device))])
 
-  # chunk data into n_lbs pieces
+  # new ring reduce
   factor = next((f for f in [32, 16, 8, 4, 2] if numel % f == 0), 1)
   base, left = (numel // factor) // n_lbs, (numel // factor) % n_lbs
-  chunks = list(itertools.pairwise(itertools.accumulate([(base + 1) * factor] * left + [base * factor] * (n_lbs - left), initial=0)))
+  chunk_sizes = [(base + 1) * factor] * left + [base * factor] * (n_lbs - left)
+  chunks = list(itertools.pairwise(itertools.accumulate(chunk_sizes, initial=0)))
 
-  # reduce-scatter
+  # extract chunks and scatter-reduce
   reduced_chunks = []
   for i,(s,e) in enumerate(chunks):
-    if use_all2all:
-      chunks_on_i = [buf.mselect(j).reshape((numel,)).shrink(((s,e),)).copy_to_device(buf.device[i]) for j in range(n_lbs)]
-      reduced_chunks.append(functools.reduce(lambda x,y: x.alu(red.arg, y), chunks_on_i))
-    else:
-      chunk, reduced = buf.reshape((numel,)).shrink(((s,e),)), buf.reshape((numel,)).shrink(((s,e),))
-      for step in range(n_lbs-1):
-        src, dest = (i+step)%n_lbs, (i+step+1)%n_lbs
-        cp = reduced.copy_to_device(buf.device[dest], src if isinstance(reduced.device, tuple) else None)
-        reduced = cp.alu(red.arg, chunk.copy_to_device(buf.device[dest], dest))
-      reduced_chunks.append(reduced)
+    chunk = buf.reshape((numel,)).shrink(((s,e),))
+    reduced_chunk = chunk
+    for step in range(n_lbs-1):
+      src, dest = (i+step)%n_lbs, (i+step+1)%n_lbs
+      # copy the chunk from the src device to the dest (operating device), and select the chunk on the dest device
+      reduced_chunk = reduced_chunk.copy_to_device(buf.device[dest], src if isinstance(reduced_chunk.device, tuple) else None) \
+        .alu(red.arg, chunk.copy_to_device(buf.device[dest], dest))
+    reduced_chunks.append(reduced_chunk)
 
   # allgather
   copied_chunks = []
-  for i,rc in enumerate(reduced_chunks):
-    if use_all2all: copied_chunks.append(UOp(Ops.MSTACK, buf.dtype, tuple(rc.copy_to_device(buf.device[j]) for j in range(n_lbs))))
-    else:
-      this_chunk: list[UOp|None] = [None] * n_lbs
-      this_chunk[(i+n_lbs-1)%n_lbs] = rc
-      for step in range(n_lbs-1):
-        this_chunk[(i+step)%n_lbs] = rc = rc.copy_to_device(buf.device[(i+step)%n_lbs])
-      copied_chunks.append(UOp(Ops.MSTACK, buf.dtype, tuple(cast(list[UOp], this_chunk))))
+  for i,c in enumerate(reduced_chunks):
+    this_chunk: list[UOp|None] = [None] * len(buf.device)
+    this_chunk[(i+len(buf.device)-1)%n_lbs] = c
+    for step in range(n_lbs-1):
+      dest = (i+step)%n_lbs
+      this_chunk[dest] = c = c.copy_to_device(buf.device[dest])
+    copied_chunks.append(UOp(Ops.MSTACK, buf.dtype, tuple(cast(list[UOp], this_chunk))))
 
   # reassemble
   pads = [((s,numel-e),) for s,e in chunks]

tinygrad/uop/validate.py

@@ -1,6 +1,6 @@
 from typing import Callable, cast
 from tinygrad.uop.ops import PatternMatcher, UPat, GroupOp, Ops, UOp, python_alu
-from tinygrad.dtype import ImageDType, dtypes, Invalid, PtrDType
+from tinygrad.dtype import ImageDType, dtypes, Invalid
 from tinygrad.helpers import IGNORE_OOB, cpu_profile
 
 try:
@@ -11,8 +11,9 @@ try:
   # IDIV is truncated division but z3 does euclidian division (floor if b>0 ceil otherwise); mod by power of two sometimes uses Ops.AND
   def z3_cdiv(a, b):return z3.If((a<0), z3.If(0<b, (a+(b-1))/b, (a-(b+1))/b), a/b)
   def z3_xor(a,b):
-    assert isinstance(a, z3.BoolRef), f"{type(a)=}, {a=}"
-    return a^b
+    if isinstance(a, z3.BoolRef): return a^b
+    assert a==-1 or b==-1, "xor can only be used in indexing if one of the arguments is -1"
+    return -a-1 if b==-1 else -b-1
   z3_alu: dict[Ops, Callable] = python_alu | {Ops.MOD: lambda a,b: a-z3_cdiv(a,b)*b, Ops.IDIV: z3_cdiv, Ops.SHR: lambda a,b: a/(2**b.as_long()),
     Ops.SHL: lambda a,b: a*(2**b.as_long()), Ops.AND: lambda a,b: a%(b+1) if isinstance(b, z3.ArithRef) else a&b, Ops.WHERE: z3.If, Ops.XOR: z3_xor,
     Ops.MAX: lambda a,b: z3.If(a<b, b, a),}
@@ -33,6 +34,7 @@ try:
     (UPat(Ops.CONST, dtypes.ints+(dtypes.index,), name="x"), lambda x,ctx: (z3.IntVal(x.arg, ctx=ctx[0].ctx), None)),
     (UPat(Ops.CONST, dtypes.bool, name="x"), lambda x,ctx: (z3.BoolVal(x.arg, ctx=ctx[0].ctx), None)),
     # casts from floats create new variables
+    (UPat(Ops.CAST, dtypes.bool, src=(UPat(dtype=dtypes.floats),), name="x"), lambda x,ctx: (z3.Bool(f"cast{len(ctx[1])}",ctx=ctx[0].ctx), None)),
     (UPat(Ops.CAST, dtypes.ints+(dtypes.index,), src=(UPat(dtype=dtypes.floats),), name="x"), lambda x,ctx:
       create_bounded(f"cast{len(ctx[1])}", x.dtype.min, x.dtype.max, ctx[0])),
     # A comparison between floats introduces a new bool variable
@@ -65,12 +67,10 @@ def validate_index(buf:UOp, idx:UOp, gate:UOp|None=None):
   # We can use UOp min/max to do a faster check, but it can give false positive since its not an exact bound and doesn't consider the mask
   if 0<=idx.vmin and idx.vmax<sz: return True
 
-  # TODO: validate these
-  # WEBGPU has a BITCAST in the index, PTX casts pointer to long
-  for x in idx.toposort() | gate.toposort():
-    if x.op is Ops.BITCAST or (x.op is Ops.CAST and isinstance(x.src[0].dtype, PtrDType)): return True
+  # WEBGPU has a BITCAST in the index. TODO: fix
+  if any(x.op is Ops.BITCAST for x in idx.toposort()): return True
 
-  if not z3_imported: raise ImportError("bounds checking requires z3 >= 4.12.4, use IGNORE_OOB=1 to disable, or \"pip install 'z3-solver>=4.12.4\"")
+  if not z3_imported: raise ImportError("z3 >= 4.12.4 is required for bounds checking, try IGNORE_OOB=0 or \"pip install 'z3-solver>=4.12.4\"")
   solver = z3.Solver(ctx=z3.Context())
   z3_idx, z3_mask = uops_to_z3(solver, idx, gate)
   solver.add(z3_mask)