tests pass

* command line interface for sqtt viz

* cleanup

* api surface area

* this confuses the llms

* document

.github/workflows/benchmark.yml

@@ -429,13 +429,15 @@ jobs:
     #    LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 BIG=2 TORCHCUDA=1 python3 test/speed/external_test_speed_v_torch.py | tee torch_speed.txt
     - name: Test speed vs theoretical
       run: AMD=1 IGNORE_BEAM_CACHE=1 CCACHE=0 BEAM_DEBUG=1 DEBUG=1 python -m pytest -rA test/external/speed_v_theoretical.py --durations=20
-    - name: Test tensor cores
-      run: |
-        AMD=1 AMD_LLVM=0 python3 test/opt/test_tensor_cores.py
-        AMD=1 AMD_LLVM=1 python3 test/opt/test_tensor_cores.py
-        AMD=1 SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
+    - name: Test tensor cores AMD_LLVM=0
+      run: AMD=1 AMD_LLVM=0 python3 test/opt/test_tensor_cores.py
+    # TODO: this is flaky
+    # - name: Test tensor cores AMD_LLVM=1
+    #   run: AMD=1 AMD_LLVM=1 python3 test/opt/test_tensor_cores.py
     - name: Run Tensor Core GEMM (AMD)
-      run: AMD=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py | tee matmul_amd.txt
+      run: |
+        AMD=1 SHOULD_USE_TC=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py
+        AMD=1 SHOULD_USE_TC=1 HALF=1 DEBUG=2 ATOL=2e-2 python3 extra/gemm/simple_matmul.py | tee matmul_amd.txt
     - name: Test AMD=1
       run: DEBUG=2 AMD=1 python -m pytest -rA test/test_tiny.py
     #- name: Test HIP=1

.github/workflows/test.yml

@@ -241,9 +241,8 @@ jobs:
       run: |
         python -m mypy --strict-equality --lineprecision-report .
         cat lineprecision.txt
-    # broken because of UPatAny
-    #- name: Run TYPED=1
-    #  run: TYPED=1 python -c "import tinygrad"
+    - name: Run TYPED=1
+      run: TYPED=1 python -c "import tinygrad"
 
   unittest:
     name: Unit Tests
@@ -642,7 +641,7 @@ jobs:
         if: matrix.backend=='amdllvm'
         run: python test/device/test_amd_llvm.py
       - name: Run pytest (amd)
-        run: python -m pytest -n=auto test/test_ops.py test/test_dtype.py test/test_dtype_alu.py test/test_linearizer.py test/test_randomness.py test/test_jit.py test/test_graph.py test/test_multitensor.py test/device/test_hcq.py --durations=20
+        run: python -m pytest -n=auto test/test_ops.py test/test_dtype.py test/test_dtype_alu.py test/test_linearizer.py test/test_randomness.py test/test_jit.py test/test_graph.py test/test_multitensor.py test/device/test_hcq.py test/testextra/test_cfg_viz.py --durations=20
       - name: Run pytest (amd)
         run: python -m pytest test/external/external_test_am.py --durations=20
       - name: Run TRANSCENDENTAL math
@@ -655,6 +654,37 @@ jobs:
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
 
+  testrdna3:
+    name: AMD ASM IDE
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Setup Environment
+        uses: ./.github/actions/setup-tinygrad
+        with:
+          key: rdna3-emu
+          deps: testing_minimal
+          amd: 'true'
+      - name: Install LLVM 21
+        run: |
+          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+          echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+          sudo apt-get update
+          sudo apt-get install llvm-21 llvm-21-tools cloc
+      - name: RDNA3 Line Count
+        run: cloc --by-file extra/assembly/amd/*.py
+      - name: Run RDNA3 emulator tests
+        run: python -m pytest -n=auto extra/assembly/amd/ --durations 20
+      - name: Install pdfplumber
+        run: pip install pdfplumber
+      - name: Verify AMD autogen is up to date
+        run: |
+          python -m extra.assembly.amd.dsl --arch all
+          python -m extra.assembly.amd.pcode --arch all
+          git diff --exit-code extra/assembly/amd/autogen/
+
   testnvidia:
     strategy:
       fail-fast: false

CLAUDE.md

@@ -34,33 +34,6 @@ result = graph_rewrite(uop, pm)
 ### Schedule Cache
 Schedules are cached by graph structure. BIND nodes (variables with bound values) are unbound before cache key computation so different values hit the same cache.
 
-## Directory Structure
-
-```
-tinygrad/
-├── tensor.py          # Tensor class, user API
-├── device.py          # Buffer, device management
-├── dtype.py           # Data types
-├── helpers.py         # Utilities, environment vars
-├── uop/
-│   ├── ops.py         # UOp class, Ops enum, PatternMatcher
-│   ├── spec.py        # UOp type verification
-│   └── symbolic.py    # Symbolic math simplification
-├── engine/
-│   ├── schedule.py    # Schedule creation, caching
-│   ├── realize.py     # Tensor realization
-│   ├── jit.py         # JIT compilation
-│   └── memory.py      # Memory planning
-├── schedule/
-│   ├── rangeify.py    # Convert movements to ranges
-│   └── indexing.py    # Index calculations
-├── codegen/
-│   ├── kernel.py      # Kernel optimization
-│   └── uopgraph.py    # UOp graph transformations
-├── renderer/          # Code generation (CUDA, Metal, etc.)
-└── runtime/           # Device backends
-```
-
 ## Testing
 
 ```bash
@@ -79,7 +52,7 @@ VIZ=1 python -c "from tinygrad import Tensor; Tensor.ones(10).sum().realize()"
 
 ## Common Environment Variables
 
-- `DEBUG=1-4` - Increasing verbosity
+- `DEBUG=1-7` - Increasing verbosity (7 shows assembly output)
 - `VIZ=1` - Enable graph visualization
 - `SPEC=1` - Enable UOp spec verification
 - `NOOPT=1` - Disable optimizations
@@ -100,6 +73,16 @@ VIZ=1 python -c "from tinygrad import Tensor; Tensor.ones(10).sum().realize()"
 - Run tests before proposing commits
 - Test with `SPEC=2` when modifying UOp-related code
 
+## Auto-generated Files (DO NOT EDIT)
+
+The following files are auto-generated and should never be edited manually:
+- `extra/assembly/amd/autogen/{arch}/__init__.py` - Generated by `python -m extra.assembly.amd.dsl --arch {arch}`
+- `extra/assembly/amd/autogen/{arch}/gen_pcode.py` - Generated by `python -m extra.assembly.amd.pcode --arch {arch}`
+
+Where `{arch}` is one of: `rdna3`, `rdna4`, `cdna`
+
+To add missing instruction implementations, add them to `extra/assembly/amd/emu.py` instead.
+
 ## Style Notes
 
 - 2-space indentation, 150 char line limit

docs/developer/developer.md

@@ -13,7 +13,7 @@ There's also a [doc describing speed](../developer/speed.md)
 
 Everything in [Tensor](../tensor/index.md) is syntactic sugar around constructing a graph of [UOps](../developer/uop.md).
 
-The `UOp` graph specifies the compute in terms of low level tinygrad ops. Not all UOps will actually become realized. There's two types of UOps, base and view. base contains compute into a contiguous buffer, and view is a view (specified by a ShapeTracker). Inputs to a base can be either base or view, inputs to a view can only be a single base.
+The `UOp` graph specifies the compute in terms of low level tinygrad ops. Not all UOps will actually become realized. There's two types of UOps, base and view. base contains compute into a contiguous buffer, and view is a view. Inputs to a base can be either base or view, inputs to a view can only be a single base.
 
 ## Scheduling
 

docs/developer/layout.md

@@ -26,9 +26,9 @@ Transforms the ast into an optimized ast. This is where BEAM search and heuristi
 
 ## tinygrad/codegen
 
-Transform the optimized ast into a linearized list of UOps.
+Transform the optimized ast into a linearized and rendered program.
 
-::: tinygrad.codegen.full_rewrite
+::: tinygrad.codegen.get_program
     options:
         members: false
         show_labels: false

eslint.config.mjs

@@ -1,9 +0,0 @@
-import globals from "globals";
-import pluginJs from "@eslint/js";
-import pluginHtml from "eslint-plugin-html";
-
-export default [
-  {files: ["**/*.html"], plugins: {html: pluginHtml}, rules:{"max-len": ["error", {"code": 150}]}},
-  {languageOptions: {globals: globals.browser}},
-  pluginJs.configs.recommended,
-];

examples/mlperf/dataloader.py

@@ -763,48 +763,26 @@ class BlendedGPTDataset:
 
     return dataset_idx, dataset_sample_idx
 
-def batch_load_llama3(bs:int, samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True):
+def get_llama3_dataset(samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True, small:bool=False) -> BlendedGPTDataset:
+  if small:
+    if val:
+      return BlendedGPTDataset(
+        [base_dir / "c4-validation-91205-samples.en_text_document"], [1.0], samples, seqlen, seed, shuffle=False)
+    return BlendedGPTDataset(
+      [base_dir / "c4-train.en_6_text_document"], [1.0], samples, seqlen, seed, shuffle=True)
   if val:
-    dataset = BlendedGPTDataset([
-      base_dir / "validation" / "c4-validationn-91205-samples.en_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, False)
-  else:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-train.en_6_text_document",
-      base_dir / "c4-train.en_7_text_document",
-    ], [
-      1.0, 1.0
-    ], samples, seqlen, seed, True)
+    return BlendedGPTDataset(
+      [base_dir / "validation" / "c4-validationn-91205-samples.en_text_document"], [1.0], samples, seqlen, seed, shuffle=False)
+  return BlendedGPTDataset(
+    [base_dir / "c4-train.en_6_text_document", base_dir / "c4-train.en_7_text_document"], [1.0, 1.0], samples, seqlen, seed, shuffle=True)
 
-  for b in range(math.ceil(samples / bs)):
-    batch = []
-    for i in range(bs):
-      tokens = dataset.get(b * bs + i)
-      batch.append(tokens)
+def iterate_llama3_dataset(dataset:BlendedGPTDataset, bs:int):
+  for b in range(math.ceil(dataset.samples / bs)):
+    batch = [dataset.get(b * bs + i) for i in range(bs)]
     yield Tensor.stack(batch, dim=0)
 
-def batch_load_llama3_small(bs:int, samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True):
-  if val:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-validation-91205-samples.en_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, False)
-  else:
-    dataset = BlendedGPTDataset([
-      base_dir / "c4-train.en_6_text_document",
-    ], [
-      1.0
-    ], samples, seqlen, seed, True)
-
-  for b in range(math.ceil(samples / bs)):
-    batch = []
-    for i in range(bs):
-      tokens = dataset.get(b * bs + i)
-      batch.append(tokens)
-    yield Tensor.stack(batch, dim=0)
+def batch_load_llama3(bs:int, samples:int, seqlen:int, base_dir:Path, seed:int=0, val:bool=True, small:bool=False):
+  return iterate_llama3_dataset(get_llama3_dataset(samples, seqlen, base_dir, seed, val, small), bs)
 
 if __name__ == "__main__":
   def load_unet3d(val):

examples/mlperf/model_eval.py

@@ -234,12 +234,9 @@ def eval_llama3():
     loss = logits.sparse_categorical_crossentropy(tokens[:, 1:])
     return loss.flatten().float()
 
-  if SMALL:
-    from examples.mlperf.dataloader import batch_load_llama3_small
-    iter = batch_load_llama3_small(BS, 5760, SEQLEN, BASEDIR, val=True)
-  else:
-    from examples.mlperf.dataloader import batch_load_llama3
-    iter = batch_load_llama3(BS, 5760, SEQLEN, BASEDIR, val=True)
+  from examples.mlperf.dataloader import get_llama3_dataset, iterate_llama3_dataset
+  eval_dataset = get_llama3_dataset(5760, SEQLEN, BASEDIR, val=True, small=bool(SMALL))
+  iter = iterate_llama3_dataset(eval_dataset, BS)
 
   losses = []
   for tokens in tqdm(iter, total=5760//BS):

examples/mlperf/model_train.py

@@ -1314,6 +1314,13 @@ def train_llama3():
   opt_base_learning_rate = getenv("LR", 8e-5 * GBS / 1152)  # NOTE: cannot change for benchmark
   opt_end_learning_rate = getenv("END_LR", 8e-7)
 
+  # ** init wandb **
+  WANDB = getenv("WANDB")
+  if WANDB:
+    import wandb
+    wandb_args = {"id": wandb_id, "resume": "must"} if (wandb_id := getenv("WANDB_RESUME", "")) else {}
+    wandb.init(config=config, **wandb_args, project="MLPerf-LLaMA3")
+
   model_params = MODEL_PARAMS[getenv("LLAMA3_SIZE", "8B")]["args"]
   # vocab_size from the mixtral tokenizer
   if not SMALL: model_params |= {"vocab_size": 32000}
@@ -1417,49 +1424,56 @@ def train_llama3():
     if getenv("FAKEDATA", 0):
       return fake_data(BS, SAMPLES)
     else:
-      if SMALL:
-        from examples.mlperf.dataloader import batch_load_llama3_small
-        return batch_load_llama3_small(BS, SAMPLES, SEQLEN, BASEDIR, seed=SEED, val=bool(TRAIN_ON_VAL))
-      else:
-        from examples.mlperf.dataloader import batch_load_llama3
-        return batch_load_llama3(BS, SAMPLES, SEQLEN, BASEDIR, seed=SEED, val=bool(TRAIN_ON_VAL))
+      from examples.mlperf.dataloader import batch_load_llama3
+      return batch_load_llama3(BS, SAMPLES, SEQLEN, BASEDIR, seed=SEED, val=bool(TRAIN_ON_VAL), small=bool(SMALL))
+
+  if getenv("FAKEDATA", 0):
+    eval_dataset = None
+  else:
+    from examples.mlperf.dataloader import get_llama3_dataset
+    eval_dataset = get_llama3_dataset(5760, SEQLEN, BASEDIR, val=True, small=bool(SMALL))
 
   def get_eval_iter():
-    if getenv("FAKEDATA", 0):
+    if eval_dataset is None:
       return fake_data(EVAL_BS, 5760)
-    else:
-      if SMALL:
-        from examples.mlperf.dataloader import batch_load_llama3_small
-        return batch_load_llama3_small(EVAL_BS, 5760, SEQLEN, BASEDIR, val=True)
-      else:
-        from examples.mlperf.dataloader import batch_load_llama3
-        return batch_load_llama3(EVAL_BS, 5760, SEQLEN, BASEDIR, val=True)
+    from examples.mlperf.dataloader import iterate_llama3_dataset
+    return iterate_llama3_dataset(eval_dataset, EVAL_BS)
 
   iter = get_train_iter()
   i, sequences_seen = resume_ckpt, 0
   for tokens in tqdm(iter, total=SAMPLES//GBS):
-    t = time.perf_counter()
     GlobalCounters.reset()
-    loss, lr = train_step(model, tokens)
-    loss = loss.float().item()
+    if getenv("TRAIN", 1):
+      t = time.perf_counter()
+      loss, lr = train_step(model, tokens)
+      loss = loss.float().item()
+      lr = lr.item()
 
-    i += 1
-    sequences_seen += tokens.shape[0]
+      i += 1
+      sequences_seen += tokens.shape[0]
 
-    tqdm.write(f"{loss:.4f} loss, {lr.item():.12f} LR, {GlobalCounters.mem_used / 1e9:.2f} GB used, {time.perf_counter()-t:.2f} s")
-    if (fname:=getenv("LOSS_FILE", "")):
-      with open(fname, "a") as f:
-        f.write(f"{i} {loss:.4f} {lr.item():.12f} {GlobalCounters.mem_used / 1e9:.2f}\n")
+      sec = time.perf_counter()-t
+      mem_gb = GlobalCounters.mem_used / 1e9
+      gflops = GlobalCounters.global_ops / 1e9 / sec
+      tqdm.write(
+        f"{i:5} {sec:.2f} s run, {loss:.4f} loss, {lr:.12f} LR, {mem_gb:.2f} GB used, {gflops:9.2f} GFLOPS")
 
-    if (ckpt_freq := getenv("CKPT")) and (i % ckpt_freq == 0 and (i != 1 or ckpt_freq == 1)):
-      tqdm.write("saving checkpoint")
-      if not os.path.exists(ckpt_dir := "./ckpts"): os.mkdir(ckpt_dir)
-      fn = f"{ckpt_dir}/llama3_{i}.safe"
-      safe_save(get_state_dict(model), fn)
+      if (fname:=getenv("LOSS_FILE", "")):
+        with open(fname, "a") as f:
+          f.write(f"{i} {loss:.4f} {lr:.12f} {mem_gb:.2f}\n")
 
-      tqdm.write("saving optim checkpoint")
-      fn = f"{ckpt_dir}/llama3_{i}_optim.safe"
-      safe_save(get_state_dict(scheduler), fn)
+      if WANDB:
+        wandb.log({"lr": lr, "train/loss": loss, "train/step_time": sec, "train/GFLOPS": gflops, "train/sequences_seen": sequences_seen})
+
+      if (ckpt_freq := getenv("CKPT")) and (i % ckpt_freq == 0 and (i != 1 or ckpt_freq == 1)):
+        tqdm.write("saving checkpoint")
+        if not os.path.exists(ckpt_dir := "./ckpts"): os.mkdir(ckpt_dir)
+        fn = f"{ckpt_dir}/llama3_{i}.safe"
+        safe_save(get_state_dict(model), fn)
+
+        tqdm.write("saving optim checkpoint")
+        fn = f"{ckpt_dir}/llama3_{i}_optim.safe"
+        safe_save(get_state_dict(scheduler), fn)
 
     if sequences_seen % EVAL_FREQ == 0 and (i != 1 or EVAL_FREQ == 1):
       tqdm.write(f"evaluating after {sequences_seen} sequences")
@@ -1475,6 +1489,9 @@ def train_llama3():
 
       tqdm.write(f"eval log perplexity: {log_perplexity:.4f}")
 
+      if WANDB:
+        wandb.log({"eval/log_perplexity": log_perplexity, "eval/sequences_seen": sequences_seen})
+
       if log_perplexity < EVAL_TARGET:
         tqdm.write(f"target achieved after {sequences_seen} sequences")
         if getenv("CKPT"):

extra/amdpci/am_smi.py

@@ -184,7 +184,7 @@ class SMICtx:
         if compact: return {k: temps[k] for k in ("Hotspot", "HBM") if temps.get(k, 0) != 0}
         return {k: v for k, v in temps.items() if v != 0}
       case _:
-        temps_keys = [(k, name) for k, name in dev.smu.smu_mod.c__EA_TEMP_e__enumvalues.items()
+        temps_keys = [(k, name) for k, name in dev.smu.smu_mod.TEMP_e.items()
                       if k < dev.smu.smu_mod.TEMP_COUNT and metrics.SmuMetrics.AvgTemperature[k] != 0]
         if compact: temps_keys = [(k, name) for k, name in temps_keys if k in (dev.smu.smu_mod.TEMP_HOTSPOT, dev.smu.smu_mod.TEMP_MEM)]
         return {name: metrics.SmuMetrics.AvgTemperature[k] for k, name in temps_keys}
@@ -193,7 +193,7 @@ class SMICtx:
     match dev.ip_ver[am.MP1_HWIP]:
       case (13,0,6): return {}
       case _:
-        voltage_keys = [(k, name) for k, name in dev.smu.smu_mod.c__EA_SVI_PLANE_e__enumvalues.items()
+        voltage_keys = [(k, name) for k, name in dev.smu.smu_mod.SVI_PLANE_e.items()
                         if k < dev.smu.smu_mod.SVI_PLANE_COUNT and metrics.SmuMetrics.AvgVoltage[k] != 0]
         return {name: metrics.SmuMetrics.AvgVoltage[k] for k, name in voltage_keys}
 

extra/assembly/amd/asm.py

@@ -0,0 +1,760 @@
+# RDNA3 assembler and disassembler
+from __future__ import annotations
+import re
+from extra.assembly.amd.dsl import Inst, RawImm, Reg, SrcMod, SGPR, VGPR, TTMP, s, v, ttmp, _RegFactory, FLOAT_ENC, SRC_FIELDS, unwrap
+from extra.assembly.amd.dsl import VCC_LO, VCC_HI, VCC, EXEC_LO, EXEC_HI, EXEC, SCC, M0, NULL, OFF
+
+# Decoding helpers
+SPECIAL_GPRS = {106: "vcc_lo", 107: "vcc_hi", 124: "null", 125: "m0", 126: "exec_lo", 127: "exec_hi", 253: "scc"}
+SPECIAL_DEC = {**SPECIAL_GPRS, **{v: str(k) for k, v in FLOAT_ENC.items()}}
+SPECIAL_PAIRS = {106: "vcc", 126: "exec"}  # Special register pairs (for 64-bit ops)
+# GFX11 hwreg names (IDs 16-17 are TBA - not supported, IDs 18-19 are PERF_SNAPSHOT)
+HWREG_NAMES = {1: 'HW_REG_MODE', 2: 'HW_REG_STATUS', 3: 'HW_REG_TRAPSTS', 4: 'HW_REG_HW_ID', 5: 'HW_REG_GPR_ALLOC',
+               6: 'HW_REG_LDS_ALLOC', 7: 'HW_REG_IB_STS', 15: 'HW_REG_SH_MEM_BASES', 18: 'HW_REG_PERF_SNAPSHOT_PC_LO',
+               19: 'HW_REG_PERF_SNAPSHOT_PC_HI', 20: 'HW_REG_FLAT_SCR_LO', 21: 'HW_REG_FLAT_SCR_HI',
+               22: 'HW_REG_XNACK_MASK', 23: 'HW_REG_HW_ID1', 24: 'HW_REG_HW_ID2', 25: 'HW_REG_POPS_PACKER', 28: 'HW_REG_IB_STS2'}
+HWREG_IDS = {v.lower(): k for k, v in HWREG_NAMES.items()}  # Reverse map for assembler
+MSG_NAMES = {128: 'MSG_RTN_GET_DOORBELL', 129: 'MSG_RTN_GET_DDID', 130: 'MSG_RTN_GET_TMA',
+             131: 'MSG_RTN_GET_REALTIME', 132: 'MSG_RTN_SAVE_WAVE', 133: 'MSG_RTN_GET_TBA'}
+_16BIT_TYPES = ('f16', 'i16', 'u16', 'b16')
+def _is_16bit(s: str) -> bool: return any(s.endswith(x) for x in _16BIT_TYPES)
+
+def decode_src(val: int) -> str:
+  if val <= 105: return f"s{val}"
+  if val in SPECIAL_DEC: return SPECIAL_DEC[val]
+  if 108 <= val <= 123: return f"ttmp{val - 108}"
+  if 128 <= val <= 192: return str(val - 128)
+  if 193 <= val <= 208: return str(-(val - 192))
+  if 256 <= val <= 511: return f"v{val - 256}"
+  return "lit" if val == 255 else f"?{val}"
+
+def _reg(prefix: str, base: int, cnt: int = 1) -> str: return f"{prefix}{base}" if cnt == 1 else f"{prefix}[{base}:{base+cnt-1}]"
+def _sreg(base: int, cnt: int = 1) -> str: return _reg("s", base, cnt)
+def _vreg(base: int, cnt: int = 1) -> str: return _reg("v", base, cnt)
+
+def _fmt_sdst(v: int, cnt: int = 1) -> str:
+  """Format SGPR destination with special register names."""
+  if v == 124: return "null"
+  if 108 <= v <= 123: return _reg("ttmp", v - 108, cnt)
+  if cnt > 1 and v in SPECIAL_PAIRS: return SPECIAL_PAIRS[v]
+  if cnt > 1: return _sreg(v, cnt)
+  return {126: "exec_lo", 127: "exec_hi", 106: "vcc_lo", 107: "vcc_hi", 125: "m0"}.get(v, f"s{v}")
+
+def _fmt_ssrc(v: int, cnt: int = 1) -> str:
+  """Format SGPR source with special register names and pairs."""
+  if cnt == 2:
+    if v in SPECIAL_PAIRS: return SPECIAL_PAIRS[v]
+    if v <= 105: return _sreg(v, 2)
+    if 108 <= v <= 123: return _reg("ttmp", v - 108, 2)
+  return decode_src(v)
+
+def _fmt_src_n(v: int, cnt: int) -> str:
+  """Format source with given register count (1, 2, or 4)."""
+  if cnt == 1: return decode_src(v)
+  if v >= 256: return _vreg(v - 256, cnt)
+  if v <= 105: return _sreg(v, cnt)
+  if cnt == 2 and v in SPECIAL_PAIRS: return SPECIAL_PAIRS[v]
+  if 108 <= v <= 123: return _reg("ttmp", v - 108, cnt)
+  return decode_src(v)
+
+def _fmt_src64(v: int) -> str:
+  """Format 64-bit source (VGPR pair, SGPR pair, or special pair)."""
+  return _fmt_src_n(v, 2)
+
+def _parse_sop_sizes(op_name: str) -> tuple[int, ...]:
+  """Parse dst and src sizes from SOP instruction name. Returns (dst_cnt, src0_cnt) or (dst_cnt, src0_cnt, src1_cnt)."""
+  if op_name in ('s_bitset0_b64', 's_bitset1_b64'): return (2, 1)
+  if op_name in ('s_lshl_b64', 's_lshr_b64', 's_ashr_i64', 's_bfe_u64', 's_bfe_i64'): return (2, 2, 1)
+  if op_name in ('s_bfm_b64',): return (2, 1, 1)
+  # SOPC: s_bitcmp0_b64, s_bitcmp1_b64 - 64-bit src0, 32-bit src1 (bit index)
+  if op_name in ('s_bitcmp0_b64', 's_bitcmp1_b64'): return (1, 2, 1)
+  if m := re.search(r'_(b|i|u)(32|64)_(b|i|u)(32|64)$', op_name):
+    return (2 if m.group(2) == '64' else 1, 2 if m.group(4) == '64' else 1)
+  if m := re.search(r'_(b|i|u)(32|64)$', op_name):
+    sz = 2 if m.group(2) == '64' else 1
+    return (sz, sz)
+  return (1, 1)
+
+# Waitcnt helpers (RDNA3 format: bits 15:10=vmcnt, bits 9:4=lgkmcnt, bits 3:0=expcnt)
+def waitcnt(vmcnt: int = 0x3f, expcnt: int = 0x7, lgkmcnt: int = 0x3f) -> int:
+  return (expcnt & 0x7) | ((lgkmcnt & 0x3f) << 4) | ((vmcnt & 0x3f) << 10)
+def decode_waitcnt(val: int) -> tuple[int, int, int]:
+  return (val >> 10) & 0x3f, val & 0xf, (val >> 4) & 0x3f  # vmcnt, expcnt, lgkmcnt
+
+# VOP3SD opcodes (shared encoding with VOP3 but different field layout)
+# Note: opcodes 0-255 are VOPC promoted to VOP3 - never treat as VOP3SD
+VOP3SD_OPCODES = {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}
+
+# Disassembler
+def disasm(inst: Inst) -> str:
+  op_val = unwrap(inst._values.get('op', 0))
+  cls_name = inst.__class__.__name__
+  # VOP3 and VOP3SD share encoding - check opcode to determine which
+  is_vop3sd = cls_name == 'VOP3' and op_val in VOP3SD_OPCODES
+  try:
+    from extra.assembly.amd.autogen import rdna3 as autogen
+    if is_vop3sd:
+      op_name = autogen.VOP3SDOp(op_val).name.lower()
+    else:
+      op_name = getattr(autogen, f"{cls_name}Op")(op_val).name.lower() if hasattr(autogen, f"{cls_name}Op") else f"op_{op_val}"
+  except (ValueError, KeyError): op_name = f"op_{op_val}"
+  def fmt_src(v): return f"0x{inst._literal:x}" if v == 255 and inst._literal is not None else decode_src(v)
+
+  # VOP1
+  if cls_name == 'VOP1':
+    vdst, src0 = unwrap(inst._values['vdst']), unwrap(inst._values['src0'])
+    if op_name == 'v_nop': return 'v_nop'
+    if op_name == 'v_pipeflush': return 'v_pipeflush'
+    parts = op_name.split('_')
+    is_16bit_dst = any(p in _16BIT_TYPES for p in parts[-2:-1]) or (len(parts) >= 2 and parts[-1] in _16BIT_TYPES and 'cvt' not in op_name)
+    is_16bit_src = parts[-1] in _16BIT_TYPES and 'sat_pk' not in op_name
+    _F64_OPS = ('v_ceil_f64', 'v_floor_f64', 'v_fract_f64', 'v_frexp_mant_f64', 'v_rcp_f64', 'v_rndne_f64', 'v_rsq_f64', 'v_sqrt_f64', 'v_trunc_f64')
+    is_f64_dst = op_name in _F64_OPS or op_name in ('v_cvt_f64_f32', 'v_cvt_f64_i32', 'v_cvt_f64_u32')
+    is_f64_src = op_name in _F64_OPS or op_name in ('v_cvt_f32_f64', 'v_cvt_i32_f64', 'v_cvt_u32_f64', 'v_frexp_exp_i32_f64')
+    if op_name == 'v_readfirstlane_b32':
+      return f"v_readfirstlane_b32 {decode_src(vdst)}, v{src0 - 256 if src0 >= 256 else src0}"
+    dst_str = _vreg(vdst, 2) if is_f64_dst else f"v{vdst & 0x7f}.{'h' if vdst >= 128 else 'l'}" if is_16bit_dst else f"v{vdst}"
+    src_str = _fmt_src64(src0) if is_f64_src else f"v{(src0 - 256) & 0x7f}.{'h' if src0 >= 384 else 'l'}" if is_16bit_src and src0 >= 256 else fmt_src(src0)
+    return f"{op_name}_e32 {dst_str}, {src_str}"
+
+  # VOP2
+  if cls_name == 'VOP2':
+    vdst, src0_raw, vsrc1 = unwrap(inst._values['vdst']), unwrap(inst._values['src0']), unwrap(inst._values['vsrc1'])
+    suffix = "" if op_name == "v_dot2acc_f32_f16" else "_e32"
+    is_16bit_op = ('_f16' in op_name or '_i16' in op_name or '_u16' in op_name) and '_f32' not in op_name and '_i32' not in op_name and 'pk_' not in op_name
+    if is_16bit_op:
+      dst_str = f"v{vdst & 0x7f}.{'h' if vdst >= 128 else 'l'}"
+      src0_str = f"v{(src0_raw - 256) & 0x7f}.{'h' if src0_raw >= 384 else 'l'}" if src0_raw >= 256 else fmt_src(src0_raw)
+      vsrc1_str = f"v{vsrc1 & 0x7f}.{'h' if vsrc1 >= 128 else 'l'}"
+    else:
+      dst_str, src0_str, vsrc1_str = f"v{vdst}", fmt_src(src0_raw), f"v{vsrc1}"
+    return f"{op_name}{suffix} {dst_str}, {src0_str}, {vsrc1_str}" + (", vcc_lo" if op_name == "v_cndmask_b32" else "")
+
+  # VOPC
+  if cls_name == 'VOPC':
+    src0, vsrc1 = unwrap(inst._values['src0']), unwrap(inst._values['vsrc1'])
+    is_64bit = any(x in op_name for x in ('f64', 'i64', 'u64'))
+    is_64bit_vsrc1 = is_64bit and 'class' not in op_name
+    is_16bit = any(x in op_name for x in ('_f16', '_i16', '_u16')) and 'f32' not in op_name
+    is_cmpx = op_name.startswith('v_cmpx')  # VOPCX writes to exec, no vcc destination
+    src0_str = _fmt_src64(src0) if is_64bit else f"v{(src0 - 256) & 0x7f}.{'h' if src0 >= 384 else 'l'}" if is_16bit and src0 >= 256 else fmt_src(src0)
+    vsrc1_str = _vreg(vsrc1, 2) if is_64bit_vsrc1 else f"v{vsrc1 & 0x7f}.{'h' if vsrc1 >= 128 else 'l'}" if is_16bit else f"v{vsrc1}"
+    return f"{op_name}_e32 {src0_str}, {vsrc1_str}" if is_cmpx else f"{op_name}_e32 vcc_lo, {src0_str}, {vsrc1_str}"
+
+  # SOPP
+  if cls_name == 'SOPP':
+    simm16 = unwrap(inst._values.get('simm16', 0))
+    # No-operand instructions (simm16 is ignored)
+    no_imm_ops = ('s_endpgm', 's_barrier', 's_wakeup', 's_icache_inv', 's_ttracedata', 's_ttracedata_imm',
+                  's_wait_idle', 's_endpgm_saved', 's_code_end', 's_endpgm_ordered_ps_done')
+    if op_name in no_imm_ops: return op_name
+    if op_name == 's_waitcnt':
+      vmcnt, expcnt, lgkmcnt = decode_waitcnt(simm16)
+      parts = []
+      if vmcnt != 0x3f: parts.append(f"vmcnt({vmcnt})")
+      if expcnt != 0x7: parts.append(f"expcnt({expcnt})")
+      if lgkmcnt != 0x3f: parts.append(f"lgkmcnt({lgkmcnt})")
+      return f"s_waitcnt {' '.join(parts)}" if parts else "s_waitcnt 0"
+    if op_name == 's_delay_alu':
+      dep_names = ['VALU_DEP_1','VALU_DEP_2','VALU_DEP_3','VALU_DEP_4','TRANS32_DEP_1','TRANS32_DEP_2','TRANS32_DEP_3','FMA_ACCUM_CYCLE_1','SALU_CYCLE_1','SALU_CYCLE_2','SALU_CYCLE_3']
+      skip_names = ['SAME','NEXT','SKIP_1','SKIP_2','SKIP_3','SKIP_4']
+      id0, skip, id1 = simm16 & 0xf, (simm16 >> 4) & 0x7, (simm16 >> 7) & 0xf
+      def dep_name(v): return dep_names[v-1] if 0 < v <= len(dep_names) else str(v)
+      parts = [f"instid0({dep_name(id0)})"] if id0 else []
+      if skip: parts.append(f"instskip({skip_names[skip]})")
+      if id1: parts.append(f"instid1({dep_name(id1)})")
+      return f"s_delay_alu {' | '.join(p for p in parts if p)}" if parts else "s_delay_alu 0"
+    if op_name.startswith('s_cbranch') or op_name.startswith('s_branch'):
+      return f"{op_name} {simm16}"
+    # Most SOPP ops require immediate (s_nop, s_setkill, s_sethalt, s_sleep, s_setprio, s_sendmsg*, etc.)
+    return f"{op_name} 0x{simm16:x}"
+
+  # SMEM
+  if cls_name == 'SMEM':
+    if op_name in ('s_gl1_inv', 's_dcache_inv'): return op_name
+    sdata, sbase, soffset, offset = unwrap(inst._values['sdata']), unwrap(inst._values['sbase']), unwrap(inst._values['soffset']), unwrap(inst._values.get('offset', 0))
+    glc, dlc = unwrap(inst._values.get('glc', 0)), unwrap(inst._values.get('dlc', 0))
+    # Format offset: "soffset offset:X" if both, "0x{offset:x}" if only imm, or decode_src(soffset)
+    off_str = f"{decode_src(soffset)} offset:0x{offset:x}" if offset and soffset != 124 else f"0x{offset:x}" if offset else decode_src(soffset)
+    sbase_idx, sbase_cnt = sbase * 2, 4 if (8 <= op_val <= 12 or op_name == 's_atc_probe_buffer') else 2
+    sbase_str = _fmt_ssrc(sbase_idx, sbase_cnt) if sbase_cnt == 2 else _sreg(sbase_idx, sbase_cnt) if sbase_idx <= 105 else _reg("ttmp", sbase_idx - 108, sbase_cnt)
+    if op_name in ('s_atc_probe', 's_atc_probe_buffer'): return f"{op_name} {sdata}, {sbase_str}, {off_str}"
+    width = {0:1, 1:2, 2:4, 3:8, 4:16, 8:1, 9:2, 10:4, 11:8, 12:16}.get(op_val, 1)
+    mods = [m for m in ["glc" if glc else "", "dlc" if dlc else ""] if m]
+    return f"{op_name} {_fmt_sdst(sdata, width)}, {sbase_str}, {off_str}" + (" " + " ".join(mods) if mods else "")
+
+  # FLAT
+  if cls_name == 'FLAT':
+    vdst, addr, data, saddr, offset, seg = [unwrap(inst._values.get(f, 0)) for f in ['vdst', 'addr', 'data', 'saddr', 'offset', 'seg']]
+    instr = f"{['flat', 'scratch', 'global'][seg] if seg < 3 else 'flat'}_{op_name.split('_', 1)[1] if '_' in op_name else op_name}"
+    width = {'b32':1, 'b64':2, 'b96':3, 'b128':4, 'u8':1, 'i8':1, 'u16':1, 'i16':1}.get(op_name.split('_')[-1], 1)
+    addr_str = _vreg(addr, 2) if saddr == 0x7F else _vreg(addr)
+    saddr_str = "" if saddr == 0x7F else f", {_sreg(saddr, 2)}" if saddr < 106 else ", off" if saddr == 124 else f", {decode_src(saddr)}"
+    off_str = f" offset:{offset}" if offset else ""
+    vdata_str = _vreg(data if 'store' in op_name else vdst, width)
+    return f"{instr} {addr_str}, {vdata_str}{saddr_str}{off_str}" if 'store' in op_name else f"{instr} {vdata_str}, {addr_str}{saddr_str}{off_str}"
+
+  # VOP3: vector ops with modifiers (can be 1, 2, or 3 sources depending on opcode range)
+  if cls_name == 'VOP3':
+    # Handle VOP3SD opcodes (same encoding, different field layout)
+    if is_vop3sd:
+      vdst = unwrap(inst._values.get('vdst', 0))
+      # VOP3SD: sdst is at bits [14:8], but VOP3 decodes opsel at [14:11], abs at [10:8], clmp at [15]
+      # We need to reconstruct sdst from these fields
+      opsel_raw = unwrap(inst._values.get('opsel', 0))
+      abs_raw = unwrap(inst._values.get('abs', 0))
+      clmp_raw = unwrap(inst._values.get('clmp', 0))
+      sdst = (clmp_raw << 7) | (opsel_raw << 3) | abs_raw
+      src0, src1, src2 = [unwrap(inst._values.get(f, 0)) for f in ('src0', 'src1', 'src2')]
+      neg = unwrap(inst._values.get('neg', 0))
+      omod = unwrap(inst._values.get('omod', 0))
+      omod_str = {1: " mul:2", 2: " mul:4", 3: " div:2"}.get(omod, "")
+      is_f64 = 'f64' in op_name
+      # v_mad_i64_i32/v_mad_u64_u32: 64-bit dst and src2, 32-bit src0/src1
+      is_mad64 = 'mad_i64_i32' in op_name or 'mad_u64_u32' in op_name
+      def fmt_sd_src(v, neg_bit, is_64bit=False):
+        s = _fmt_src64(v) if (is_64bit or is_f64) else fmt_src(v)
+        return f"-{s}" if neg_bit else s
+      src0_str, src1_str = fmt_sd_src(src0, neg & 1), fmt_sd_src(src1, neg & 2)
+      src2_str = fmt_sd_src(src2, neg & 4, is_mad64)
+      dst_str = _vreg(vdst, 2) if (is_f64 or is_mad64) else f"v{vdst}"
+      sdst_str = _fmt_sdst(sdst, 1)
+      # v_add_co_u32, v_sub_co_u32, v_subrev_co_u32, v_add_co_ci_u32, etc. only use 2 sources
+      if op_name in ('v_add_co_u32', 'v_sub_co_u32', 'v_subrev_co_u32', 'v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'):
+        return f"{op_name} {dst_str}, {sdst_str}, {src0_str}, {src1_str}"
+      # v_div_scale uses 3 sources
+      return f"{op_name} {dst_str}, {sdst_str}, {src0_str}, {src1_str}, {src2_str}" + omod_str
+
+    vdst = unwrap(inst._values.get('vdst', 0))
+    src0, src1, src2 = [unwrap(inst._values.get(f, 0)) for f in ('src0', 'src1', 'src2')]
+    neg, abs_, clmp = unwrap(inst._values.get('neg', 0)), unwrap(inst._values.get('abs', 0)), unwrap(inst._values.get('clmp', 0))
+    opsel = unwrap(inst._values.get('opsel', 0))
+    # Check if 64-bit op (needs register pairs)
+    is_f64 = 'f64' in op_name or 'i64' in op_name or 'u64' in op_name or 'b64' in op_name
+    # v_cmp_class_* has 64-bit src0 but 32-bit src1 (class mask)
+    is_class = 'class' in op_name
+    # Shift ops: v_*rev_*64 have 32-bit shift amount (src0), 64-bit value (src1)
+    is_shift64 = 'rev' in op_name and '64' in op_name and op_name.startswith('v_')
+    # v_ldexp_f64: 64-bit src0 (mantissa), 32-bit src1 (exponent)
+    is_ldexp64 = op_name == 'v_ldexp_f64'
+    # v_trig_preop_f64: 64-bit dst/src0, 32-bit src1 (exponent/scale)
+    is_trig_preop = op_name == 'v_trig_preop_f64'
+    # v_readlane_b32: destination is SGPR (despite vdst field)
+    is_readlane = op_name == 'v_readlane_b32'
+    # SAD/QSAD/MQSAD instructions have mixed sizes
+    # v_qsad_pk_u16_u8, v_mqsad_pk_u16_u8: 64-bit dst/src0/src2, 32-bit src1
+    # v_mqsad_u32_u8: 128-bit (4 reg) dst/src2, 64-bit src0, 32-bit src1
+    is_sad64 = any(x in op_name for x in ('qsad_pk', 'mqsad_pk'))
+    is_mqsad_u32 = 'mqsad_u32' in op_name
+    # Detect 16-bit and 64-bit operand sizes for various instruction patterns
+    if 'cvt_pk' in op_name:
+      is_f16_dst, is_f16_src, is_f16_src2 = False, op_name.endswith('16'), False
+    elif m := re.match(r'v_(?:cvt|frexp_exp)_([a-z0-9_]+)_([a-z0-9]+)', op_name):
+      dst_type, src_type = m.group(1), m.group(2)
+      is_f16_dst, is_f16_src, is_f16_src2 = _is_16bit(dst_type), _is_16bit(src_type), _is_16bit(src_type)
+      is_f64_dst, is_f64_src, is_f64 = '64' in dst_type, '64' in src_type, False
+    elif re.match(r'v_mad_[iu]32_[iu]16', op_name):
+      is_f16_dst, is_f16_src, is_f16_src2 = False, True, False  # 32-bit dst, 16-bit src0/src1, 32-bit src2
+    elif 'pack_b32' in op_name:
+      is_f16_dst, is_f16_src, is_f16_src2 = False, True, True  # 32-bit dst, 16-bit sources
+    else:
+      is_16bit_op = any(x in op_name for x in _16BIT_TYPES) and not any(x in op_name for x in ('dot2', 'pk_', 'sad', 'msad', 'qsad', 'mqsad'))
+      is_f16_dst = is_f16_src = is_f16_src2 = is_16bit_op
+    # Check if any opsel bit is set (any operand uses .h) - if so, we need explicit .l for low-half
+    any_hi = opsel != 0
+    def fmt_vop3_src(v, neg_bit, abs_bit, hi_bit=False, reg_cnt=1, is_16=False):
+      s = _fmt_src_n(v, reg_cnt) if reg_cnt > 1 else f"v{v - 256}.h" if is_16 and v >= 256 and hi_bit else f"v{v - 256}.l" if is_16 and v >= 256 and any_hi else fmt_src(v)
+      if abs_bit: s = f"|{s}|"
+      return f"-{s}" if neg_bit else s
+    # Determine register count for each source (check for cvt-specific 64-bit flags first)
+    is_src0_64 = locals().get('is_f64_src', is_f64 and not is_shift64) or is_sad64 or is_mqsad_u32
+    is_src1_64 = is_f64 and not is_class and not is_ldexp64 and not is_trig_preop
+    src0_cnt = 2 if is_src0_64 else 1
+    src1_cnt = 2 if is_src1_64 else 1
+    src2_cnt = 4 if is_mqsad_u32 else 2 if (is_f64 or is_sad64) else 1
+    src0_str = fmt_vop3_src(src0, neg & 1, abs_ & 1, opsel & 1, src0_cnt, is_f16_src)
+    src1_str = fmt_vop3_src(src1, neg & 2, abs_ & 2, opsel & 2, src1_cnt, is_f16_src)
+    src2_str = fmt_vop3_src(src2, neg & 4, abs_ & 4, opsel & 4, src2_cnt, is_f16_src2)
+    # Format destination - for 16-bit ops, use .h/.l suffix; readlane uses SGPR dest
+    is_dst_64 = locals().get('is_f64_dst', is_f64) or is_sad64
+    dst_cnt = 4 if is_mqsad_u32 else 2 if is_dst_64 else 1
+    if is_readlane:
+      dst_str = _fmt_sdst(vdst, 1)
+    elif dst_cnt > 1:
+      dst_str = _vreg(vdst, dst_cnt)
+    elif is_f16_dst:
+      dst_str = f"v{vdst}.h" if (opsel & 8) else f"v{vdst}.l" if any_hi else f"v{vdst}"
+    else:
+      dst_str = f"v{vdst}"
+    clamp_str = " clamp" if clmp else ""
+    omod = unwrap(inst._values.get('omod', 0))
+    omod_str = {1: " mul:2", 2: " mul:4", 3: " div:2"}.get(omod, "")
+    # op_sel for non-VGPR sources (when opsel bits are set but source is not a VGPR)
+    # For 16-bit ops with VGPR sources, opsel is encoded in .h/.l suffix
+    # For non-VGPR sources or non-16-bit ops, we need explicit op_sel
+    has_nonvgpr_opsel = (src0 < 256 and (opsel & 1)) or (src1 < 256 and (opsel & 2)) or (src2 < 256 and (opsel & 4))
+    need_opsel = has_nonvgpr_opsel or (opsel and not is_f16_src)
+    # Helper to format opsel string based on source count
+    def fmt_opsel(num_src):
+      if not need_opsel: return ""
+      # When dst is .h (for 16-bit ops) and non-VGPR sources have opsel, use all 1s
+      if is_f16_dst and (opsel & 8):  # dst is .h
+        return f" op_sel:[1,1,1{',1' if num_src == 3 else ''}]"
+      # Otherwise output actual opsel values
+      if num_src == 3:
+        return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{(opsel >> 2) & 1},{(opsel >> 3) & 1}]"
+      return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{(opsel >> 2) & 1}]"
+    # Determine number of sources based on opcode range:
+    # 0-255: VOPC promoted (comparison, 2 src, sdst)
+    # 256-383: VOP2 promoted (2 src)
+    # 384-511: VOP1 promoted (1 src)
+    # 512+: Native VOP3 (2 or 3 src depending on instruction)
+    if op_val < 256:  # VOPC promoted
+      # VOPCX (v_cmpx_*) writes to exec, no explicit destination
+      if op_name.startswith('v_cmpx'):
+        return f"{op_name}_e64 {src0_str}, {src1_str}"
+      return f"{op_name}_e64 {_fmt_sdst(vdst, 1)}, {src0_str}, {src1_str}"
+    elif op_val < 384:  # VOP2 promoted
+      # v_cndmask_b32 in VOP3 format has 3 sources (src2 is mask selector)
+      if 'cndmask' in op_name:
+        return f"{op_name}_e64 {dst_str}, {src0_str}, {src1_str}, {src2_str}" + fmt_opsel(3) + clamp_str + omod_str
+      return f"{op_name}_e64 {dst_str}, {src0_str}, {src1_str}" + fmt_opsel(2) + clamp_str + omod_str
+    elif op_val < 512:  # VOP1 promoted
+      if op_name in ('v_nop', 'v_pipeflush'): return f"{op_name}_e64"
+      return f"{op_name}_e64 {dst_str}, {src0_str}" + fmt_opsel(1) + clamp_str + omod_str
+    else:  # Native VOP3 - determine 2 vs 3 sources based on instruction name
+      # 3-source ops: fma, mad, min3, max3, med3, div_fixup, div_fmas, sad, msad, qsad, mqsad, lerp, alignbit/byte, cubeid/sc/tc/ma, bfe, bfi, perm_b32, permlane, cndmask
+      # Note: v_writelane_b32 is 2-src (src0, src1 with vdst as 3rd operand - read-modify-write)
+      is_3src = any(x in op_name for x in ('fma', 'mad', 'min3', 'max3', 'med3', 'div_fix', 'div_fmas', 'sad', 'lerp', 'align', 'cube',
+                                            'bfe', 'bfi', 'perm_b32', 'permlane', 'cndmask', 'xor3', 'or3', 'add3', 'lshl_or', 'and_or', 'lshl_add',
+                                            'add_lshl', 'xad', 'maxmin', 'minmax', 'dot2', 'cvt_pk_u8', 'mullit'))
+      if is_3src:
+        return f"{op_name} {dst_str}, {src0_str}, {src1_str}, {src2_str}" + fmt_opsel(3) + clamp_str + omod_str
+      return f"{op_name} {dst_str}, {src0_str}, {src1_str}" + fmt_opsel(2) + clamp_str + omod_str
+
+  # VOP3SD: 3-source with scalar destination (v_div_scale_*, v_add_co_u32, v_mad_*64_*32, etc.)
+  if cls_name == 'VOP3SD':
+    vdst, sdst = unwrap(inst._values.get('vdst', 0)), unwrap(inst._values.get('sdst', 0))
+    src0, src1, src2 = [unwrap(inst._values.get(f, 0)) for f in ('src0', 'src1', 'src2')]
+    neg, omod, clmp = unwrap(inst._values.get('neg', 0)), unwrap(inst._values.get('omod', 0)), unwrap(inst._values.get('clmp', 0))
+    is_f64, is_mad64 = 'f64' in op_name, 'mad_i64_i32' in op_name or 'mad_u64_u32' in op_name
+    def fmt_neg(v, neg_bit, is_64=False): return f"-{_fmt_src64(v) if (is_64 or is_f64) else fmt_src(v)}" if neg_bit else _fmt_src64(v) if (is_64 or is_f64) else fmt_src(v)
+    srcs = [fmt_neg(src0, neg & 1), fmt_neg(src1, neg & 2), fmt_neg(src2, neg & 4, is_mad64)]
+    dst_str, sdst_str = _vreg(vdst, 2) if (is_f64 or is_mad64) else f"v{vdst}", _fmt_sdst(sdst, 1)
+    clamp_str, omod_str = " clamp" if clmp else "", {1: " mul:2", 2: " mul:4", 3: " div:2"}.get(omod, "")
+    is_2src = op_name in ('v_add_co_u32', 'v_sub_co_u32', 'v_subrev_co_u32')
+    suffix = "_e64" if op_name.startswith('v_') and 'co_' in op_name else ""
+    return f"{op_name}{suffix} {dst_str}, {sdst_str}, {', '.join(srcs[:2] if is_2src else srcs)}" + clamp_str + omod_str
+
+  # VOPD: dual-issue instructions
+  if cls_name == 'VOPD':
+    from extra.assembly.amd.autogen import rdna3 as autogen
+    opx, opy, vdstx, vdsty_enc = [unwrap(inst._values.get(f, 0)) for f in ('opx', 'opy', 'vdstx', 'vdsty')]
+    srcx0, vsrcx1, srcy0, vsrcy1 = [unwrap(inst._values.get(f, 0)) for f in ('srcx0', 'vsrcx1', 'srcy0', 'vsrcy1')]
+    vdsty = (vdsty_enc << 1) | ((vdstx & 1) ^ 1)  # Decode vdsty
+    def fmt_vopd(op, vdst, src0, vsrc1):
+      try: name = autogen.VOPDOp(op).name.lower()
+      except (ValueError, KeyError): name = f"op_{op}"
+      return f"{name} v{vdst}, {fmt_src(src0)}" if 'mov' in name else f"{name} v{vdst}, {fmt_src(src0)}, v{vsrc1}"
+    return f"{fmt_vopd(opx, vdstx, srcx0, vsrcx1)} :: {fmt_vopd(opy, vdsty, srcy0, vsrcy1)}"
+
+  # VOP3P: packed vector ops
+  if cls_name == 'VOP3P':
+    vdst, clmp = unwrap(inst._values.get('vdst', 0)), unwrap(inst._values.get('clmp', 0))
+    src0, src1, src2 = [unwrap(inst._values.get(f, 0)) for f in ('src0', 'src1', 'src2')]
+    neg, neg_hi = unwrap(inst._values.get('neg', 0)), unwrap(inst._values.get('neg_hi', 0))
+    opsel, opsel_hi, opsel_hi2 = unwrap(inst._values.get('opsel', 0)), unwrap(inst._values.get('opsel_hi', 0)), unwrap(inst._values.get('opsel_hi2', 0))
+    is_wmma, is_3src = 'wmma' in op_name, any(x in op_name for x in ('fma', 'mad', 'dot', 'wmma'))
+    def fmt_bits(name, val, n): return f"{name}:[{','.join(str((val >> i) & 1) for i in range(n))}]"
+    # WMMA: f16/bf16 use 8-reg sources, iu8 uses 4-reg, iu4 uses 2-reg; all have 8-reg dst
+    if is_wmma:
+      src_cnt = 2 if 'iu4' in op_name else 4 if 'iu8' in op_name else 8
+      src0_str, src1_str, src2_str = _fmt_src_n(src0, src_cnt), _fmt_src_n(src1, src_cnt), _fmt_src_n(src2, 8)
+      dst_str = _vreg(vdst, 8)
+    else:
+      src0_str, src1_str, src2_str = _fmt_src_n(src0, 1), _fmt_src_n(src1, 1), _fmt_src_n(src2, 1)
+      dst_str = f"v{vdst}"
+    n = 3 if is_3src else 2
+    full_opsel_hi = opsel_hi | (opsel_hi2 << 2)
+    mods = [fmt_bits("op_sel", opsel, n)] if opsel else []
+    if full_opsel_hi != (0b111 if is_3src else 0b11): mods.append(fmt_bits("op_sel_hi", full_opsel_hi, n))
+    if neg: mods.append(fmt_bits("neg_lo", neg, n))
+    if neg_hi: mods.append(fmt_bits("neg_hi", neg_hi, n))
+    if clmp: mods.append("clamp")
+    mod_str = " " + " ".join(mods) if mods else ""
+    return f"{op_name} {dst_str}, {src0_str}, {src1_str}, {src2_str}{mod_str}" if is_3src else f"{op_name} {dst_str}, {src0_str}, {src1_str}{mod_str}"
+
+  # VINTERP: interpolation instructions
+  if cls_name == 'VINTERP':
+    vdst = unwrap(inst._values.get('vdst', 0))
+    src0, src1, src2 = [unwrap(inst._values.get(f, 0)) for f in ('src0', 'src1', 'src2')]
+    neg, waitexp, clmp = unwrap(inst._values.get('neg', 0)), unwrap(inst._values.get('waitexp', 0)), unwrap(inst._values.get('clmp', 0))
+    def fmt_neg_vi(v, neg_bit): return f"-{v}" if neg_bit else v
+    srcs = [fmt_neg_vi(f"v{s - 256}" if s >= 256 else fmt_src(s), neg & (1 << i)) for i, s in enumerate([src0, src1, src2])]
+    mods = [m for m in [f"wait_exp:{waitexp}" if waitexp else "", "clamp" if clmp else ""] if m]
+    return f"{op_name} v{vdst}, {', '.join(srcs)}" + (" " + " ".join(mods) if mods else "")
+
+  # MUBUF/MTBUF helpers
+  def _buf_vaddr(vaddr, offen, idxen): return _vreg(vaddr, 2) if offen and idxen else f"v{vaddr}" if offen or idxen else "off"
+  def _buf_srsrc(srsrc): srsrc_base = srsrc * 4; return _reg("ttmp", srsrc_base - 108, 4) if 108 <= srsrc_base <= 123 else _sreg(srsrc_base, 4)
+
+  # MUBUF: buffer load/store
+  if cls_name == 'MUBUF':
+    vdata, vaddr, srsrc, soffset = [unwrap(inst._values.get(f, 0)) for f in ('vdata', 'vaddr', 'srsrc', 'soffset')]
+    offset, offen, idxen = unwrap(inst._values.get('offset', 0)), unwrap(inst._values.get('offen', 0)), unwrap(inst._values.get('idxen', 0))
+    glc, dlc, slc, tfe = [unwrap(inst._values.get(f, 0)) for f in ('glc', 'dlc', 'slc', 'tfe')]
+    if op_name in ('buffer_gl0_inv', 'buffer_gl1_inv'): return op_name
+    # Determine data width from op name
+    if 'd16' in op_name: width = 2 if any(x in op_name for x in ('xyz', 'xyzw')) else 1
+    elif 'atomic' in op_name:
+      base_width = 2 if any(x in op_name for x in ('b64', 'u64', 'i64')) else 1
+      width = base_width * 2 if 'cmpswap' in op_name else base_width
+    else: width = {'b32':1, 'b64':2, 'b96':3, 'b128':4, 'b16':1, 'x':1, 'xy':2, 'xyz':3, 'xyzw':4}.get(op_name.split('_')[-1], 1)
+    if tfe: width += 1
+    mods = [m for m in ["offen" if offen else "", "idxen" if idxen else "", f"offset:{offset}" if offset else "",
+                        "glc" if glc else "", "dlc" if dlc else "", "slc" if slc else "", "tfe" if tfe else ""] if m]
+    return f"{op_name} {_vreg(vdata, width)}, {_buf_vaddr(vaddr, offen, idxen)}, {_buf_srsrc(srsrc)}, {decode_src(soffset)}" + (" " + " ".join(mods) if mods else "")
+
+  # MTBUF: typed buffer load/store
+  if cls_name == 'MTBUF':
+    vdata, vaddr, srsrc, soffset = [unwrap(inst._values.get(f, 0)) for f in ('vdata', 'vaddr', 'srsrc', 'soffset')]
+    offset, tbuf_fmt, offen, idxen = [unwrap(inst._values.get(f, 0)) for f in ('offset', 'format', 'offen', 'idxen')]
+    glc, dlc, slc = [unwrap(inst._values.get(f, 0)) for f in ('glc', 'dlc', 'slc')]
+    mods = [f"format:{tbuf_fmt}"] + [m for m in ["idxen" if idxen else "", "offen" if offen else "", f"offset:{offset}" if offset else "",
+                                                  "glc" if glc else "", "dlc" if dlc else "", "slc" if slc else ""] if m]
+    width = 2 if 'd16' in op_name and any(x in op_name for x in ('xyz', 'xyzw')) else 1 if 'd16' in op_name else {'x':1, 'xy':2, 'xyz':3, 'xyzw':4}.get(op_name.split('_')[-1], 1)
+    return f"{op_name} {_vreg(vdata, width)}, {_buf_vaddr(vaddr, offen, idxen)}, {_buf_srsrc(srsrc)}, {decode_src(soffset)} {' '.join(mods)}"
+
+  # SOP1/SOP2/SOPC/SOPK
+  if cls_name in ('SOP1', 'SOP2', 'SOPC', 'SOPK'):
+    sizes = _parse_sop_sizes(op_name)
+    dst_cnt, src0_cnt = sizes[0], sizes[1]
+    src1_cnt = sizes[2] if len(sizes) > 2 else src0_cnt
+    if cls_name == 'SOP1':
+      sdst, ssrc0 = unwrap(inst._values.get('sdst', 0)), unwrap(inst._values.get('ssrc0', 0))
+      if op_name == 's_getpc_b64': return f"{op_name} {_fmt_sdst(sdst, 2)}"
+      if op_name in ('s_setpc_b64', 's_rfe_b64'): return f"{op_name} {_fmt_ssrc(ssrc0, 2)}"
+      if op_name == 's_swappc_b64': return f"{op_name} {_fmt_sdst(sdst, 2)}, {_fmt_ssrc(ssrc0, 2)}"
+      if op_name in ('s_sendmsg_rtn_b32', 's_sendmsg_rtn_b64'):
+        return f"{op_name} {_fmt_sdst(sdst, 2 if 'b64' in op_name else 1)}, sendmsg({MSG_NAMES.get(ssrc0, str(ssrc0))})"
+      ssrc0_str = fmt_src(ssrc0) if src0_cnt == 1 else _fmt_ssrc(ssrc0, src0_cnt)
+      return f"{op_name} {_fmt_sdst(sdst, dst_cnt)}, {ssrc0_str}"
+    if cls_name == 'SOP2':
+      sdst, ssrc0, ssrc1 = [unwrap(inst._values.get(f, 0)) for f in ('sdst', 'ssrc0', 'ssrc1')]
+      ssrc0_str = fmt_src(ssrc0) if ssrc0 == 255 else _fmt_ssrc(ssrc0, src0_cnt)
+      ssrc1_str = fmt_src(ssrc1) if ssrc1 == 255 else _fmt_ssrc(ssrc1, src1_cnt)
+      return f"{op_name} {_fmt_sdst(sdst, dst_cnt)}, {ssrc0_str}, {ssrc1_str}"
+    if cls_name == 'SOPC':
+      return f"{op_name} {_fmt_ssrc(unwrap(inst._values.get('ssrc0', 0)), src0_cnt)}, {_fmt_ssrc(unwrap(inst._values.get('ssrc1', 0)), src1_cnt)}"
+    if cls_name == 'SOPK':
+      sdst, simm16 = unwrap(inst._values.get('sdst', 0)), unwrap(inst._values.get('simm16', 0))
+      if op_name == 's_version': return f"{op_name} 0x{simm16:x}"
+      if op_name in ('s_setreg_b32', 's_getreg_b32'):
+        hwreg_id, hwreg_offset, hwreg_size = simm16 & 0x3f, (simm16 >> 6) & 0x1f, ((simm16 >> 11) & 0x1f) + 1
+        hwreg_str = f"0x{simm16:x}" if hwreg_id in (16, 17) else f"hwreg({HWREG_NAMES.get(hwreg_id, str(hwreg_id))}, {hwreg_offset}, {hwreg_size})"
+        return f"{op_name} {hwreg_str}, {_fmt_sdst(sdst, 1)}" if op_name == 's_setreg_b32' else f"{op_name} {_fmt_sdst(sdst, 1)}, {hwreg_str}"
+      return f"{op_name} {_fmt_sdst(sdst, dst_cnt)}, 0x{simm16:x}"
+
+  # Generic fallback
+  def fmt_field(n, v):
+    v = unwrap(v)
+    if n in SRC_FIELDS: return fmt_src(v) if v != 255 else "0xff"
+    if n in ('sdst', 'vdst'): return f"{'s' if n == 'sdst' else 'v'}{v}"
+    return f"v{v}" if n == 'vsrc1' else f"0x{v:x}" if n == 'simm16' else str(v)
+  ops = [fmt_field(n, inst._values.get(n, 0)) for n in inst._fields if n not in ('encoding', 'op')]
+  return f"{op_name} {', '.join(ops)}" if ops else op_name
+
+# Assembler
+SPECIAL_REGS = {'vcc_lo': RawImm(106), 'vcc_hi': RawImm(107), 'vcc': RawImm(106), 'null': RawImm(124), 'off': RawImm(124), 'm0': RawImm(125),
+                'exec_lo': RawImm(126), 'exec_hi': RawImm(127), 'exec': RawImm(126), 'scc': RawImm(253), 'src_scc': RawImm(253)}
+FLOAT_CONSTS = {'0.5': 0.5, '-0.5': -0.5, '1.0': 1.0, '-1.0': -1.0, '2.0': 2.0, '-2.0': -2.0, '4.0': 4.0, '-4.0': -4.0}
+REG_MAP: dict[str, _RegFactory] = {'s': s, 'v': v, 't': ttmp, 'ttmp': ttmp}
+
+def parse_operand(op: str) -> tuple:
+  op = op.strip().lower()
+  neg = op.startswith('-') and not op[1:2].isdigit(); op = op[1:] if neg else op
+  abs_ = op.startswith('|') and op.endswith('|') or op.startswith('abs(') and op.endswith(')')
+  op = op[1:-1] if op.startswith('|') else op[4:-1] if op.startswith('abs(') else op
+  hi_half = op.endswith('.h')
+  op = re.sub(r'\.[lh]$', '', op)
+  if op in FLOAT_CONSTS: return (FLOAT_CONSTS[op], neg, abs_, hi_half)
+  if re.match(r'^-?\d+$', op): return (int(op), neg, abs_, hi_half)
+  if m := re.match(r'^-?0x([0-9a-f]+)$', op):
+    v = -int(m.group(1), 16) if op.startswith('-') else int(m.group(1), 16)
+    return (v, neg, abs_, hi_half)
+  if op in SPECIAL_REGS: return (SPECIAL_REGS[op], neg, abs_, hi_half)
+  if op == 'lit': return (RawImm(255), neg, abs_, hi_half)  # literal marker (actual value comes from literal word)
+  if m := re.match(r'^([svt](?:tmp)?)\[(\d+):(\d+)\]$', op): return (REG_MAP[m.group(1)][int(m.group(2)):int(m.group(3))], neg, abs_, hi_half)
+  if m := re.match(r'^([svt](?:tmp)?)(\d+)$', op):
+    reg = REG_MAP[m.group(1)][int(m.group(2))]
+    reg.hi = hi_half
+    return (reg, neg, abs_, hi_half)
+  # hwreg(name, offset, size) or hwreg(name) -> simm16 encoding
+  if m := re.match(r'^hwreg\((\w+)(?:,\s*(\d+),\s*(\d+))?\)$', op):
+    name_str = m.group(1).lower()
+    hwreg_id = HWREG_IDS.get(name_str, int(name_str) if name_str.isdigit() else None)
+    if hwreg_id is None: raise ValueError(f"unknown hwreg name: {name_str}")
+    offset, size = int(m.group(2)) if m.group(2) else 0, int(m.group(3)) if m.group(3) else 32
+    return (((size - 1) << 11) | (offset << 6) | hwreg_id, neg, abs_, hi_half)
+  raise ValueError(f"cannot parse operand: {op}")
+
+SMEM_OPS = {'s_load_b32', 's_load_b64', 's_load_b128', 's_load_b256', 's_load_b512',
+            's_buffer_load_b32', 's_buffer_load_b64', 's_buffer_load_b128', 's_buffer_load_b256', 's_buffer_load_b512'}
+SOP1_SRC_ONLY = {'s_setpc_b64', 's_rfe_b64'}
+SOP1_MSG_IMM = {'s_sendmsg_rtn_b32', 's_sendmsg_rtn_b64'}
+SOPK_IMM_ONLY = {'s_version'}
+SOPK_IMM_FIRST = {'s_setreg_b32'}
+SOPK_UNSUPPORTED = {'s_setreg_imm32_b32'}
+
+def _operand_to_dsl(op: str) -> str:
+  """Transform a single operand from LLVM assembly syntax to DSL expression string."""
+  op = op.strip()
+  # Handle negation prefix
+  neg = False
+  if op.startswith('-') and not (op[1:2].isdigit() or (len(op) > 2 and op[1] == '0' and op[2] in 'xX')):
+    neg, op = True, op[1:]
+  # Handle abs modifier: |x| or abs(x)
+  abs_ = False
+  if op.startswith('|') and op.endswith('|'):
+    abs_, op = True, op[1:-1]
+  elif op.startswith('abs(') and op.endswith(')'):
+    abs_, op = True, op[4:-1]
+  # Handle .h/.l suffix for 16-bit ops
+  hi_suffix = ""
+  if op.endswith('.h'): hi_suffix, op = ".h", op[:-2]
+  elif op.endswith('.l'): hi_suffix, op = ".l", op[:-2]
+  op_lower = op.lower()
+
+  # Helper to apply modifiers
+  def apply_mods(base: str) -> str:
+    if not neg and not abs_: return f"{base}{hi_suffix}"
+    if abs_: return f"{'-' if neg else ''}abs({base}){hi_suffix}"
+    return f"-{base}{hi_suffix}"
+
+  # Special registers - vcc maps to VCC_LO (64-bit alias)
+  special_map = {'vcc_lo': 'VCC_LO', 'vcc_hi': 'VCC_HI', 'vcc': 'VCC_LO', 'null': 'NULL', 'off': 'OFF',
+                 'm0': 'M0', 'exec_lo': 'EXEC_LO', 'exec_hi': 'EXEC_HI', 'exec': 'EXEC_LO', 'scc': 'SCC',
+                 'src_scc': 'SCC'}
+  if op_lower in special_map: return apply_mods(special_map[op_lower])
+  # Float constants
+  float_map = {'0.5': '0.5', '-0.5': '-0.5', '1.0': '1.0', '-1.0': '-1.0', '2.0': '2.0', '-2.0': '-2.0', '4.0': '4.0', '-4.0': '-4.0'}
+  if op in float_map: return apply_mods(float_map[op])
+  # Register range: v[0:3], s[4:7]
+  if m := re.match(r'^([svt](?:tmp)?)\[(\d+):(\d+)\]$', op_lower):
+    prefix = {'s': 's', 'v': 'v', 't': 'ttmp', 'ttmp': 'ttmp'}[m.group(1)]
+    return apply_mods(f"{prefix}[{m.group(2)}:{m.group(3)}]")
+  # Single register: v0, s1, ttmp5
+  if m := re.match(r'^([svt](?:tmp)?)(\d+)$', op_lower):
+    prefix = {'s': 's', 'v': 'v', 't': 'ttmp', 'ttmp': 'ttmp'}[m.group(1)]
+    return apply_mods(f"{prefix}[{m.group(2)}]")
+  # Integer literals (decimal or hex) - use SrcMod wrapper when modifiers present
+  if re.match(r'^-?\d+$', op) or re.match(r'^-?0x([0-9a-fA-F]+)$', op):
+    if neg or abs_:
+      return f"SrcMod({op}, neg={neg}, abs_={abs_})"
+    return op
+  # hwreg(name, offset, size) -> pass through
+  if op_lower.startswith('hwreg('): return apply_mods(op)
+  # sendmsg(...) -> pass through
+  if op_lower.startswith('sendmsg('): return apply_mods(op)
+  # Fallback: return as-is
+  return apply_mods(op)
+
+def _parse_operands(op_str: str) -> list[str]:
+  """Parse comma-separated operands, respecting brackets and pipes."""
+  operands, current, depth, in_pipe = [], "", 0, False
+  for ch in op_str:
+    if ch in '[(': depth += 1
+    elif ch in '])': depth -= 1
+    elif ch == '|': in_pipe = not in_pipe
+    if ch == ',' and depth == 0 and not in_pipe:
+      operands.append(current.strip())
+      current = ""
+    else:
+      current += ch
+  if current.strip(): operands.append(current.strip())
+  return operands
+
+def _unwrap_dsl(s: str) -> str:
+  """Unwrap a DSL expression to get the raw value for literals."""
+  if re.match(r'^-?\d+$', s): return s
+  if re.match(r'^-?0x[0-9a-fA-F]+$', s): return s
+  return s
+
+def get_dsl(text: str) -> str:
+  """Transform LLVM-style assembly instruction to Python DSL expression string."""
+  text = text.strip()
+  # Extract and remove trailing modifiers (must happen before operand parsing)
+  kwargs = []
+  # Extract mul:N and div:N modifiers (omod)
+  omod_val = 0
+  if m := re.search(r'\s+mul:2(?:\s|$)', text, re.I):
+    omod_val = 1; text = text[:m.start()] + text[m.end():]
+  elif m := re.search(r'\s+mul:4(?:\s|$)', text, re.I):
+    omod_val = 2; text = text[:m.start()] + text[m.end():]
+  elif m := re.search(r'\s+div:2(?:\s|$)', text, re.I):
+    omod_val = 3; text = text[:m.start()] + text[m.end():]
+  if omod_val: kwargs.append(f'omod={omod_val}')
+  # Extract clamp modifier
+  if m := re.search(r'\s+clamp(?:\s|$)', text, re.I):
+    kwargs.append('clmp=1')
+    text = text[:m.start()] + text[m.end():]
+  # Extract op_sel:[...] modifier - interpretation depends on format:
+  # VOP3: [src0, src1, dst] or [src0, src1, src2, dst] -> bits 0, 1, (2), 3
+  # VOP3P/WMMA: [src0, src1, src2] -> bits 0, 1, 2 (no dst bit, 3-source ops)
+  opsel_explicit = None
+  if m := re.search(r'\s+op_sel:\[([^\]]+)\]', text, re.I):
+    bits = [int(x.strip()) for x in m.group(1).split(',')]
+    # Check if this is a VOP3P instruction (v_pk_*, v_wmma_*, v_dot*)
+    mnemonic = text.split()[0].lower()
+    is_vop3p = mnemonic.startswith(('v_pk_', 'v_wmma_', 'v_dot'))
+    if len(bits) == 3:
+      if is_vop3p:
+        # VOP3P: [src0, src1, src2] -> bits 0, 1, 2
+        opsel_explicit = bits[0] | (bits[1] << 1) | (bits[2] << 2)
+      else:
+        # VOP3: [src0, src1, dst] -> bits 0, 1, 3
+        opsel_explicit = bits[0] | (bits[1] << 1) | (bits[2] << 3)
+    else:
+      opsel_explicit = sum(b << i for i, b in enumerate(bits))
+    text = text[:m.start()] + text[m.end():]
+  if m := re.search(r'\s+wait_exp:(\d+)', text, re.I):
+    kwargs.append(f'waitexp={m.group(1)}')
+    text = text[:m.start()] + text[m.end():]
+  # Extract offset:N for FLAT/GLOBAL/SCRATCH/SMEM (can be hex or decimal)
+  offset_val = None
+  if m := re.search(r'\s+offset:(0x[0-9a-fA-F]+|-?\d+)', text, re.I):
+    offset_val = m.group(1)
+    text = text[:m.start()] + text[m.end():]
+  # Extract dlc modifier (before glc to avoid partial match issues)
+  dlc_val = None
+  if m := re.search(r'\s+dlc(?:\s|$)', text, re.I):
+    dlc_val = 1
+    text = text[:m.start()] + text[m.end():]
+  # Extract glc modifier
+  glc_val = None
+  if m := re.search(r'\s+glc(?:\s|$)', text, re.I):
+    glc_val = 1
+    text = text[:m.start()] + text[m.end():]
+  # Extract neg_lo:[...] and neg_hi:[...] for VOP3P
+  neg_lo_val = None
+  if m := re.search(r'\s+neg_lo:\[([^\]]+)\]', text, re.I):
+    bits = [int(x.strip()) for x in m.group(1).split(',')]
+    neg_lo_val = sum(b << i for i, b in enumerate(bits))
+    text = text[:m.start()] + text[m.end():]
+  neg_hi_val = None
+  if m := re.search(r'\s+neg_hi:\[([^\]]+)\]', text, re.I):
+    bits = [int(x.strip()) for x in m.group(1).split(',')]
+    neg_hi_val = sum(b << i for i, b in enumerate(bits))
+    text = text[:m.start()] + text[m.end():]
+  parts = text.replace(',', ' ').split()
+  if not parts: raise ValueError("empty instruction")
+  mnemonic, op_str = parts[0].lower(), text[len(parts[0]):].strip()
+  # Handle s_waitcnt specially
+  if mnemonic == 's_waitcnt':
+    vmcnt, expcnt, lgkmcnt = 0x3f, 0x7, 0x3f
+    for part in op_str.replace(',', ' ').split():
+      if m := re.match(r'vmcnt\((\d+)\)', part): vmcnt = int(m.group(1))
+      elif m := re.match(r'expcnt\((\d+)\)', part): expcnt = int(m.group(1))
+      elif m := re.match(r'lgkmcnt\((\d+)\)', part): lgkmcnt = int(m.group(1))
+      elif re.match(r'^0x[0-9a-f]+$|^\d+$', part): return f"s_waitcnt(simm16={int(part, 0)})"
+    wc = waitcnt(vmcnt, expcnt, lgkmcnt)
+    return f"s_waitcnt(simm16={wc})"
+  # Handle VOPD dual-issue: opx dst, src :: opy dst, src
+  if '::' in text:
+    x_part, y_part = text.split('::')
+    x_parts, y_parts = x_part.strip().replace(',', ' ').split(), y_part.strip().replace(',', ' ').split()
+    opx_name, opy_name = x_parts[0].upper(), y_parts[0].upper()
+    x_ops = [_operand_to_dsl(p) for p in x_parts[1:]]
+    y_ops = [_operand_to_dsl(p) for p in y_parts[1:]]
+    vdstx, srcx0 = x_ops[0], x_ops[1] if len(x_ops) > 1 else '0'
+    vsrcx1 = x_ops[2] if len(x_ops) > 2 else 'v[0]'
+    vdsty, srcy0 = y_ops[0], y_ops[1] if len(y_ops) > 1 else '0'
+    vsrcy1 = y_ops[2] if len(y_ops) > 2 else 'v[0]'
+    lit = None
+    if 'fmaak' in opx_name.lower() and len(x_ops) > 3: lit = x_ops[3]
+    elif 'fmamk' in opx_name.lower() and len(x_ops) > 3: lit, vsrcx1 = x_ops[2], x_ops[3]
+    elif 'fmaak' in opy_name.lower() and len(y_ops) > 3: lit = y_ops[3]
+    elif 'fmamk' in opy_name.lower() and len(y_ops) > 3: lit, vsrcy1 = y_ops[2], y_ops[3]
+    lit_str = f", literal={lit}" if lit else ""
+    return f"VOPD(VOPDOp.{opx_name}, VOPDOp.{opy_name}, vdstx={vdstx}, vdsty={vdsty}, srcx0={srcx0}, vsrcx1={vsrcx1}, srcy0={srcy0}, vsrcy1={vsrcy1}{lit_str})"
+  operands = _parse_operands(op_str)
+  dsl_args = [_operand_to_dsl(op) for op in operands]
+  # Handle special instructions
+  if mnemonic in SOPK_UNSUPPORTED: raise ValueError(f"unsupported instruction: {mnemonic}")
+  if mnemonic in SOP1_SRC_ONLY: return f"{mnemonic}(ssrc0={dsl_args[0]})"
+  if mnemonic in SOP1_MSG_IMM: return f"{mnemonic}(sdst={dsl_args[0]}, ssrc0=RawImm({_unwrap_dsl(dsl_args[1])}))"
+  if mnemonic in SOPK_IMM_ONLY: return f"{mnemonic}(simm16={dsl_args[0]})"
+  if mnemonic in SOPK_IMM_FIRST: return f"{mnemonic}(simm16={dsl_args[0]}, sdst={dsl_args[1]})"
+  # SMEM with immediate offset (offset in operand[2] or offset: modifier)
+  if mnemonic in SMEM_OPS:
+    glc_str = ", glc=1" if glc_val else ""
+    dlc_str = ", dlc=1" if dlc_val else ""
+    # Pure immediate offset in operand[2]
+    if len(operands) >= 3 and re.match(r'^-?[0-9]|^-?0x', operands[2].strip().lower()):
+      return f"{mnemonic}(sdata={dsl_args[0]}, sbase={dsl_args[1]}, offset={dsl_args[2]}, soffset=RawImm(124){glc_str}{dlc_str})"
+    # Register soffset with offset: modifier
+    if offset_val and len(operands) >= 3:
+      return f"{mnemonic}(sdata={dsl_args[0]}, sbase={dsl_args[1]}, offset={offset_val}, soffset={dsl_args[2]}{glc_str}{dlc_str})"
+    # Register soffset only (no offset modifier)
+    if len(operands) >= 3:
+      return f"{mnemonic}(sdata={dsl_args[0]}, sbase={dsl_args[1]}, soffset={dsl_args[2]}{glc_str}{dlc_str})"
+  # Buffer ops with 'off'
+  if mnemonic.startswith('buffer_') and len(operands) >= 2 and operands[1].strip().lower() == 'off':
+    soff = f"RawImm({_unwrap_dsl(dsl_args[3])})" if len(dsl_args) > 3 else "RawImm(0)"
+    return f"{mnemonic}(vdata={dsl_args[0]}, vaddr=0, srsrc={dsl_args[2]}, soffset={soff})"
+  # FLAT/GLOBAL/SCRATCH load
+  if (mnemonic.startswith('flat_load') or mnemonic.startswith('global_load') or mnemonic.startswith('scratch_load')) and len(dsl_args) >= 3:
+    off = f", offset={offset_val}" if offset_val else ""
+    return f"{mnemonic}(vdst={dsl_args[0]}, addr={dsl_args[1]}, saddr={dsl_args[2]}{off})"
+  # FLAT/GLOBAL/SCRATCH store
+  if (mnemonic.startswith('flat_store') or mnemonic.startswith('global_store') or mnemonic.startswith('scratch_store')) and len(dsl_args) >= 3:
+    off = f", offset={offset_val}" if offset_val else ""
+    return f"{mnemonic}(addr={dsl_args[0]}, data={dsl_args[1]}, saddr={dsl_args[2]}{off})"
+  # Handle v_fmaak/v_fmamk literals
+  lit_str = ""
+  if mnemonic in ('v_fmaak_f32', 'v_fmaak_f16') and len(dsl_args) == 4:
+    lit_str, dsl_args = f", literal={_unwrap_dsl(dsl_args[3])}", dsl_args[:3]
+  elif mnemonic in ('v_fmamk_f32', 'v_fmamk_f16') and len(dsl_args) == 4:
+    lit_str, dsl_args = f", literal={_unwrap_dsl(dsl_args[2])}", [dsl_args[0], dsl_args[1], dsl_args[3]]
+  # Handle v_add_co_ci_u32_e32 etc with vcc operands - strip implicit vcc sdst and carry_in, add _e32 suffix
+  vcc_ops = {'v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'}
+  if mnemonic.replace('_e32', '') in vcc_ops and len(dsl_args) >= 5:
+    mnemonic = mnemonic.replace('_e32', '') + '_e32'  # Ensure _e32 suffix for VOP2 encoding
+    dsl_args = [dsl_args[0], dsl_args[2], dsl_args[3]]
+  # v_cmp_*_e32: strip implicit vcc_lo dest
+  if mnemonic.startswith('v_cmp') and not mnemonic.endswith('_e64') and len(dsl_args) >= 3 and operands[0].strip().lower() in ('vcc_lo', 'vcc_hi', 'vcc'):
+    dsl_args = dsl_args[1:]
+  # CMPX with _e64: prepend implicit EXEC_LO (vdst=126)
+  if 'cmpx' in mnemonic and mnemonic.endswith('_e64') and len(dsl_args) == 2:
+    dsl_args = ['RawImm(126)'] + dsl_args
+  # Build the function name - use mnemonic as-is, replacing . with _
+  func_name = mnemonic.replace('.', '_')
+  # When explicit opsel is given, strip .h/.l from register args (opsel overrides)
+  if opsel_explicit is not None:
+    dsl_args = [re.sub(r'\.[hl]$', '', a) for a in dsl_args]
+  args_str = ', '.join(dsl_args)
+  all_kwargs = list(kwargs)
+  if lit_str: all_kwargs.append(lit_str.lstrip(', '))
+  if opsel_explicit is not None: all_kwargs.append(f'opsel={opsel_explicit}')
+  if neg_lo_val is not None: all_kwargs.append(f'neg={neg_lo_val}')
+  if neg_hi_val is not None: all_kwargs.append(f'neg_hi={neg_hi_val}')
+  kwargs_str = ', '.join(all_kwargs)
+  if kwargs_str:
+    return f"{func_name}({args_str}, {kwargs_str})" if args_str else f"{func_name}({kwargs_str})"
+  return f"{func_name}({args_str})"
+
+def asm(text: str) -> Inst:
+  """Assemble LLVM-style instruction text to Inst by transforming to DSL and eval."""
+  from extra.assembly.amd.autogen import rdna3 as autogen
+  dsl_expr = get_dsl(text)
+  namespace = {name: getattr(autogen, name) for name in dir(autogen) if not name.startswith('_')}
+  namespace.update({'s': s, 'v': v, 'ttmp': ttmp, 'abs': abs, 'RawImm': RawImm, 'SrcMod': SrcMod, 'VGPR': VGPR, 'SGPR': SGPR, 'TTMP': TTMP,
+                    'VCC_LO': VCC_LO, 'VCC_HI': VCC_HI, 'VCC': VCC, 'EXEC_LO': EXEC_LO, 'EXEC_HI': EXEC_HI, 'EXEC': EXEC,
+                    'SCC': SCC, 'M0': M0, 'NULL': NULL, 'OFF': OFF})
+  try:
+    return eval(dsl_expr, namespace)
+  except NameError:
+    # Try with _e32 suffix for VOP1/VOP2/VOPC (only for v_* instructions)
+    if m := re.match(r'^(v_\w+)(\(.*\))$', dsl_expr):
+      return eval(f"{m.group(1)}_e32{m.group(2)}", namespace)
+    raise

extra/assembly/amd/dsl.py

@@ -0,0 +1,640 @@
+# library for RDNA3 assembly DSL
+# mypy: ignore-errors
+from __future__ import annotations
+from enum import IntEnum
+from typing import overload, Annotated, TypeVar, Generic
+
+# Bit field DSL
+class BitField:
+  def __init__(self, hi: int, lo: int, name: str | None = None): self.hi, self.lo, self.name = hi, lo, name
+  def __set_name__(self, owner, name): self.name, self._owner = name, owner
+  def __eq__(self, val: int) -> tuple[BitField, int]: return (self, val)  # type: ignore
+  def mask(self) -> int: return (1 << (self.hi - self.lo + 1)) - 1
+  @property
+  def marker(self) -> type | None:
+    # Get marker from Annotated type hint if present
+    import typing
+    if hasattr(self, '_owner') and self.name:
+      hints = typing.get_type_hints(self._owner, include_extras=True)
+      if self.name in hints:
+        hint = hints[self.name]
+        if typing.get_origin(hint) is Annotated:
+          args = typing.get_args(hint)
+          return args[1] if len(args) > 1 else None
+    return None
+  @overload
+  def __get__(self, obj: None, objtype: type) -> BitField: ...
+  @overload
+  def __get__(self, obj: object, objtype: type | None = None) -> int: ...
+  def __get__(self, obj, objtype=None):
+    if obj is None: return self
+    val = unwrap(obj._values.get(self.name, 0))
+    # Convert to IntEnum if marker is an IntEnum subclass
+    if self.marker and isinstance(self.marker, type) and issubclass(self.marker, IntEnum):
+      try: return self.marker(val)
+      except ValueError: pass
+    return val
+
+class _Bits:
+  def __getitem__(self, key) -> BitField: return BitField(key.start, key.stop) if isinstance(key, slice) else BitField(key, key)
+bits = _Bits()
+
+# Source operand with modifiers - base class for anything that can be a src with neg/abs
+class SrcMod:
+  __slots__ = ('val', 'neg', 'abs_')
+  def __init__(self, val: int, neg: bool = False, abs_: bool = False): self.val, self.neg, self.abs_ = val, neg, abs_
+  def __repr__(self): return f"{'-' if self.neg else ''}{'|' if self.abs_ else ''}{self.val}{'|' if self.abs_ else ''}"
+  def __neg__(self): return SrcMod(self.val, not self.neg, self.abs_)
+  def __abs__(self): return SrcMod(self.val, self.neg, True)
+
+# Register types
+class Reg(SrcMod):
+  __slots__ = ('idx', 'count', 'hi')
+  def __init__(self, idx: int, count: int = 1, hi: bool = False, neg: bool = False, abs_: bool = False):
+    self.idx, self.count, self.hi = idx, count, hi
+    super().__init__(idx, neg, abs_)
+  def __repr__(self): return f"{self.__class__.__name__.lower()[0]}[{self.idx}]" if self.count == 1 else f"{self.__class__.__name__.lower()[0]}[{self.idx}:{self.idx + self.count}]"
+  def __neg__(self): return self.__class__(self.idx, self.count, self.hi, not self.neg, self.abs_)
+  def __abs__(self): return self.__class__(self.idx, self.count, self.hi, self.neg, True)
+  @property
+  def l(self): return self.__class__(self.idx, self.count, False, self.neg, self.abs_)
+  @property
+  def h(self): return self.__class__(self.idx, self.count, True, self.neg, self.abs_)
+
+T = TypeVar('T', bound=Reg)
+class _RegFactory(Generic[T]):
+  def __init__(self, cls: type[T], name: str): self._cls, self._name = cls, name
+  @overload
+  def __getitem__(self, key: int) -> Reg: ...
+  @overload
+  def __getitem__(self, key: slice) -> Reg: ...
+  def __getitem__(self, key: int | slice) -> Reg:
+    return self._cls(key.start, key.stop - key.start + 1) if isinstance(key, slice) else self._cls(key)
+  def __repr__(self): return f"<{self._name} factory>"
+
+class SGPR(Reg): pass
+class VGPR(Reg): pass
+class TTMP(Reg): pass
+s: _RegFactory[SGPR] = _RegFactory(SGPR, "SGPR")
+v: _RegFactory[VGPR] = _RegFactory(VGPR, "VGPR")
+ttmp: _RegFactory[TTMP] = _RegFactory(TTMP, "TTMP")
+
+# Special registers as SrcMod objects (support -VCC_LO, abs(EXEC_LO), etc.)
+VCC_LO, VCC_HI, VCC = SrcMod(106), SrcMod(107), SrcMod(106)
+EXEC_LO, EXEC_HI, EXEC = SrcMod(126), SrcMod(127), SrcMod(126)
+SCC, M0, NULL, OFF = SrcMod(253), SrcMod(125), SrcMod(124), SrcMod(124)
+
+# Field type markers (runtime classes for validation)
+class _SSrc: pass
+class _Src: pass
+class _Imm: pass
+class _SImm: pass
+class _VDSTYEnc: pass  # VOPD vdsty: encoded = actual >> 1, actual = (encoded << 1) | ((vdstx & 1) ^ 1)
+class _SGPRField: pass
+class _VGPRField: pass
+
+# Type aliases for annotations - tells mypy it's a BitField while preserving marker info
+SSrc = Annotated[BitField, _SSrc]
+Src = Annotated[BitField, _Src]
+Imm = Annotated[BitField, _Imm]
+SImm = Annotated[BitField, _SImm]
+VDSTYEnc = Annotated[BitField, _VDSTYEnc]
+SGPRField = Annotated[BitField, _SGPRField]
+VGPRField = Annotated[BitField, _VGPRField]
+class RawImm:
+  def __init__(self, val: int): self.val = val
+  def __repr__(self): return f"RawImm({self.val})"
+  def __eq__(self, other): return isinstance(other, RawImm) and self.val == other.val
+
+def unwrap(val) -> int:
+  if isinstance(val, RawImm): return val.val
+  if isinstance(val, SrcMod) and not isinstance(val, Reg): return val.val  # Special registers like VCC_LO, NULL
+  if hasattr(val, 'value'): return val.value  # IntEnum
+  if hasattr(val, 'idx'): return val.idx  # Reg
+  return val
+
+# Encoding helpers
+FLOAT_ENC = {0.5: 240, -0.5: 241, 1.0: 242, -1.0: 243, 2.0: 244, -2.0: 245, 4.0: 246, -4.0: 247}
+SRC_FIELDS = {'src0', 'src1', 'src2', 'ssrc0', 'ssrc1', 'soffset', 'srcx0', 'srcy0'}
+RAW_FIELDS = {'vdata', 'vdst', 'vaddr', 'addr', 'data', 'data0', 'data1', 'sdst', 'sdata', 'vsrc1'}
+
+def _encode_reg(val: Reg) -> int:
+  if isinstance(val, TTMP): return 108 + val.idx
+  return val.idx  # hi bit is handled via opsel, not in register encoding
+
+def encode_src(val) -> int:
+  if isinstance(val, VGPR): return 256 + _encode_reg(val)
+  if isinstance(val, Reg): return _encode_reg(val)
+  if isinstance(val, SrcMod) and not isinstance(val, Reg):
+    # SrcMod wraps either special registers (VCC_LO=106, EXEC_LO=126, etc.) or literals
+    # Special register values are in valid encoding ranges - return as-is
+    # Literals (large integers) need 255 marker
+    v = val.val
+    # Valid source encoding ranges: 0-127 (SGPRs/special), 128-192 (inline const), 193-208 (neg inline), 240-247 (float), 251-253 (special)
+    if 0 <= v <= 127 or 240 <= v <= 255: return v  # SGPRs, special regs, float constants
+    if 128 <= v <= 192: return v  # Inline positive constants (0-64)
+    if 193 <= v <= 208: return v  # Inline negative constants (-1 to -16)
+    return 255  # Literal marker - value stored separately
+  if hasattr(val, 'value'): return val.value  # IntEnum
+  if isinstance(val, float): return 128 if val == 0.0 else FLOAT_ENC.get(val, 255)
+  return 128 + val if isinstance(val, int) and 0 <= val <= 64 else 192 + (-val) if isinstance(val, int) and -16 <= val <= -1 else 255
+
+# Instruction base class
+class Inst:
+  _fields: dict[str, BitField]
+  _encoding: tuple[BitField, int] | None = None
+  _defaults: dict[str, int] = {}
+  _values: dict[str, int | RawImm]
+  _words: int  # size in 32-bit words, set by decode_program
+  _literal: int | None
+
+  def __init_subclass__(cls, **kwargs):
+    super().__init_subclass__(**kwargs)
+    cls._fields = {n: v[0] if isinstance(v, tuple) else v for n, v in cls.__dict__.items() if isinstance(v, BitField) or (isinstance(v, tuple) and len(v) == 2 and isinstance(v[0], BitField))}
+    if 'encoding' in cls._fields and isinstance(cls.__dict__.get('encoding'), tuple): cls._encoding = cls.__dict__['encoding']
+
+  def __init__(self, *args, literal: int | None = None, **kwargs):
+    self._values, self._literal = dict(self._defaults), literal
+    # Map positional args to field names
+    field_names = [n for n in self._fields if n != 'encoding']
+    orig_args = dict(zip(field_names, args))
+    orig_args.update(kwargs)
+    self._values.update(orig_args)
+    # Validate register counts for SMEM instructions (before encoding)
+    if self.__class__.__name__ == 'SMEM':
+      op_val = orig_args.get(field_names[0]) if args else orig_args.get('op')
+      if op_val is not None:
+        if hasattr(op_val, 'value'): op_val = op_val.value
+        expected_cnt = {0:1, 1:2, 2:4, 3:8, 4:16, 8:1, 9:2, 10:4, 11:8, 12:16}.get(op_val)
+        sdata_val = orig_args.get('sdata')
+        if expected_cnt is not None and isinstance(sdata_val, Reg) and sdata_val.count != expected_cnt:
+          raise ValueError(f"SMEM op {op_val} expects {expected_cnt} registers, got {sdata_val.count}")
+    # Validate register counts for SOP1 instructions (b32 = 1 reg, b64 = 2 regs)
+    if self.__class__.__name__ == 'SOP1':
+      op_val = orig_args.get(field_names[0]) if args else orig_args.get('op')
+      if op_val is not None and hasattr(op_val, 'name'):
+        expected = 2 if op_val.name.endswith('_B64') else 1
+        sdst_val, ssrc0_val = orig_args.get('sdst'), orig_args.get('ssrc0')
+        if isinstance(sdst_val, Reg) and sdst_val.count != expected:
+          raise ValueError(f"SOP1 {op_val.name} expects {expected} destination register(s), got {sdst_val.count}")
+        if isinstance(ssrc0_val, Reg) and ssrc0_val.count != expected:
+          raise ValueError(f"SOP1 {op_val.name} expects {expected} source register(s), got {ssrc0_val.count}")
+    # Type check and encode values
+    for name, val in list(self._values.items()):
+      if name == 'encoding': continue
+      # For RawImm, only process RAW_FIELDS to unwrap to int
+      if isinstance(val, RawImm):
+        if name in RAW_FIELDS: self._values[name] = val.val
+        continue
+      field = self._fields.get(name)
+      marker = field.marker if field else None
+      # Type validation
+      if marker is _SGPRField:
+        if isinstance(val, VGPR): raise TypeError(f"field '{name}' requires SGPR, got VGPR")
+        if not isinstance(val, (SGPR, TTMP, SrcMod, int, RawImm)): raise TypeError(f"field '{name}' requires SGPR, got {type(val).__name__}")
+      if marker is _VGPRField:
+        if not isinstance(val, VGPR): raise TypeError(f"field '{name}' requires VGPR, got {type(val).__name__}")
+      if marker is _SSrc and isinstance(val, VGPR): raise TypeError(f"field '{name}' requires scalar source, got VGPR")
+      # Encode source fields as RawImm for consistent disassembly
+      if name in SRC_FIELDS:
+        encoded = encode_src(val)
+        # For VOP1/VOP2/VOPC (no opsel field), encode hi bit in src value
+        if isinstance(val, Reg) and val.hi and 'opsel' not in self._fields:
+          encoded |= 0x80
+        self._values[name] = RawImm(encoded)
+        # Handle neg/abs/opsel modifiers for VOP3 instructions
+        if isinstance(val, SrcMod):
+          if val.neg and 'neg' in self._fields:
+            neg_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+            cur_neg = self._values.get('neg', 0)
+            self._values['neg'] = (cur_neg.val if isinstance(cur_neg, RawImm) else cur_neg) | neg_bit
+          if val.abs_ and 'abs' in self._fields:
+            abs_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+            cur_abs = self._values.get('abs', 0)
+            self._values['abs'] = (cur_abs.val if isinstance(cur_abs, RawImm) else cur_abs) | abs_bit
+        # Handle hi (opsel) for 16-bit ops - only for formats with opsel field
+        if isinstance(val, Reg) and val.hi and 'opsel' in self._fields:
+          opsel_bit = {'src0': 1, 'src1': 2, 'src2': 4}.get(name, 0)
+          cur_opsel = self._values.get('opsel', 0)
+          self._values['opsel'] = (cur_opsel.val if isinstance(cur_opsel, RawImm) else cur_opsel) | opsel_bit
+        # Track literal value if needed (encoded as 255)
+        # For 64-bit ops, store literal in high 32 bits (to match from_bytes decoding and to_bytes encoding)
+        if encoded == 255 and self._literal is None:
+          if isinstance(val, SrcMod) and not isinstance(val, Reg):
+            # SrcMod wrapping a literal value
+            self._literal = (val.val << 32) if self._is_64bit_op() else val.val
+          elif isinstance(val, int) and not isinstance(val, IntEnum):
+            self._literal = (val << 32) if self._is_64bit_op() else val
+          elif isinstance(val, float):
+            import struct
+            lit32 = struct.unpack('<I', struct.pack('<f', val))[0]
+            self._literal = (lit32 << 32) if self._is_64bit_op() else lit32
+      # Encode raw register fields for consistent repr
+      elif name in RAW_FIELDS:
+        if isinstance(val, Reg):
+          encoded = _encode_reg(val)
+          # For VOP1/VOP2/VOPC (no opsel field), encode hi bit in register value
+          if val.hi and 'opsel' not in self._fields:
+            encoded |= 0x80
+          self._values[name] = encoded
+          # Handle vdst hi (opsel bit 3) for 16-bit ops - only for formats with opsel field
+          if name == 'vdst' and val.hi and 'opsel' in self._fields:
+            cur_opsel = self._values.get('opsel', 0)
+            self._values['opsel'] = (cur_opsel.val if isinstance(cur_opsel, RawImm) else cur_opsel) | 8
+        elif hasattr(val, 'value'): self._values[name] = val.value  # IntEnum like SrcEnum.NULL
+      # Encode sbase (divided by 2) and srsrc/ssamp (divided by 4)
+      elif name == 'sbase':
+        if isinstance(val, Reg): self._values[name] = val.idx // 2
+        elif isinstance(val, SrcMod): self._values[name] = val.val // 2  # Special regs like VCC_LO
+      elif name in {'srsrc', 'ssamp'} and isinstance(val, Reg):
+        self._values[name] = val.idx // 4
+      # VOPD vdsty: encode as actual >> 1 (constraint: vdsty parity must be opposite of vdstx)
+      elif marker is _VDSTYEnc and isinstance(val, VGPR):
+        self._values[name] = val.idx >> 1
+
+  def _encode_field(self, name: str, val) -> int:
+    if isinstance(val, RawImm): return val.val
+    if isinstance(val, SrcMod) and not isinstance(val, Reg): return val.val  # Special regs like VCC_LO
+    if name in {'srsrc', 'ssamp'}: return val.idx // 4 if isinstance(val, Reg) else val
+    if name == 'sbase': return val.idx // 2 if isinstance(val, Reg) else val.val // 2 if isinstance(val, SrcMod) else val
+    if name in RAW_FIELDS: return _encode_reg(val) if isinstance(val, Reg) else val
+    if isinstance(val, Reg) or name in SRC_FIELDS: return encode_src(val)
+    return val.value if hasattr(val, 'value') else val
+
+  def to_int(self) -> int:
+    word = (self._encoding[1] & self._encoding[0].mask()) << self._encoding[0].lo if self._encoding else 0
+    for n, bf in self._fields.items():
+      if n != 'encoding' and n in self._values: word |= (self._encode_field(n, self._values[n]) & bf.mask()) << bf.lo
+    return word
+
+  def _get_literal(self) -> int | None:
+    for n in SRC_FIELDS:
+      if n in self._values and not isinstance(v := self._values[n], RawImm) and isinstance(v, int) and not isinstance(v, IntEnum) and not (0 <= v <= 64 or -16 <= v <= -1): return v
+    return None
+
+  def _is_64bit_op(self) -> bool:
+    """Check if this instruction uses 64-bit operands (and thus 64-bit literals).
+    Exception: V_LDEXP_F64 has 32-bit integer src1, so its literal is 32-bit."""
+    op = self._values.get('op')
+    if op is None: return False
+    # op may be an enum (from __init__) or an int (from from_int)
+    op_name = op.name if hasattr(op, 'name') else None
+    if op_name is None and self.__class__.__name__ == 'VOP3':
+      from extra.assembly.amd.autogen.rdna3 import VOP3Op
+      try: op_name = VOP3Op(op).name
+      except ValueError: pass
+    if op_name is None: return False
+    # V_LDEXP_F64 has 32-bit integer exponent in src1, so literal is 32-bit
+    if op_name == 'V_LDEXP_F64': return False
+    return op_name.endswith(('_F64', '_B64', '_I64', '_U64'))
+
+  def to_bytes(self) -> bytes:
+    result = self.to_int().to_bytes(self._size(), 'little')
+    lit = self._get_literal() or getattr(self, '_literal', None)
+    if lit is None: return result
+    # For 64-bit ops, literal is stored in high 32 bits internally, but encoded as 4 bytes
+    lit32 = (lit >> 32) if self._is_64bit_op() else lit
+    return result + (lit32 & 0xffffffff).to_bytes(4, 'little')
+
+  @classmethod
+  def _size(cls) -> int: return 4 if issubclass(cls, Inst32) else 8
+  def size(self) -> int:
+    # Literal is always 4 bytes in the binary (for 64-bit ops, it's in high 32 bits)
+    return self._size() + (4 if self._literal is not None else 0)
+
+  @classmethod
+  def from_int(cls, word: int):
+    inst = object.__new__(cls)
+    inst._values = {n: RawImm(v) if n in SRC_FIELDS else v for n, bf in cls._fields.items() if n != 'encoding' for v in [(word >> bf.lo) & bf.mask()]}
+    inst._literal = None
+    return inst
+
+  @classmethod
+  def from_bytes(cls, data: bytes):
+    inst = cls.from_int(int.from_bytes(data[:cls._size()], 'little'))
+    op_val = inst._values.get('op', 0)
+    has_literal = cls.__name__ == 'VOP2' and op_val in (44, 45, 55, 56)
+    has_literal = has_literal or (cls.__name__ == 'SOP2' and op_val in (69, 70))
+    for n in SRC_FIELDS:
+      if n in inst._values and isinstance(inst._values[n], RawImm) and inst._values[n].val == 255: has_literal = True
+    if has_literal:
+      # For 64-bit ops, the literal is 32 bits placed in the HIGH 32 bits of the 64-bit value
+      # (low 32 bits are zero). This is how AMD hardware interprets 32-bit literals for 64-bit ops.
+      if len(data) >= cls._size() + 4:
+        lit32 = int.from_bytes(data[cls._size():cls._size()+4], 'little')
+        inst._literal = (lit32 << 32) if inst._is_64bit_op() else lit32
+    return inst
+
+  def __repr__(self):
+    # Use _fields order and exclude fields that are 0/default (for consistent repr after roundtrip)
+    def is_zero(v): return (isinstance(v, int) and v == 0) or (isinstance(v, VGPR) and v.idx == 0 and v.count == 1)
+    items = [(k, self._values[k]) for k in self._fields if k in self._values and k != 'encoding'
+             and not (is_zero(self._values[k]) and k not in {'op'})]
+    lit = f", literal={hex(self._literal)}" if self._literal is not None else ""
+    return f"{self.__class__.__name__}({', '.join(f'{k}={v}' for k, v in items)}{lit})"
+
+  def __eq__(self, other):
+    if not isinstance(other, Inst): return NotImplemented
+    return self.__class__ == other.__class__ and self._values == other._values and self._literal == other._literal
+
+  def __hash__(self): return hash((self.__class__.__name__, tuple(sorted((k, repr(v)) for k, v in self._values.items())), self._literal))
+
+  def disasm(self) -> str:
+    from extra.assembly.amd.asm import disasm
+    return disasm(self)
+
+class Inst32(Inst): pass
+class Inst64(Inst): pass
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CODE GENERATION: generates autogen/__init__.py by parsing AMD ISA PDFs
+# Supports both RDNA3.5 and CDNA4 instruction set PDFs - auto-detects format
+# ═══════════════════════════════════════════════════════════════════════════════
+
+PDF_URLS = {
+  "rdna3": "https://docs.amd.com/api/khub/documents/UVVZM22UN7tMUeiW_4ShTQ/content",  # RDNA3.5
+  "rdna4": "https://docs.amd.com/api/khub/documents/uQpkEvk3pv~kfAb2x~j4uw/content",
+  "cdna": ["https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-mi300-cdna3-instruction-set-architecture.pdf",
+           "https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf"],
+}
+FIELD_TYPES = {'SSRC0': 'SSrc', 'SSRC1': 'SSrc', 'SOFFSET': 'SSrc', 'SADDR': 'SSrc', 'SRC0': 'Src', 'SRC1': 'Src', 'SRC2': 'Src',
+  'SDST': 'SGPRField', 'SBASE': 'SGPRField', 'SDATA': 'SGPRField', 'SRSRC': 'SGPRField', 'VDST': 'VGPRField', 'VSRC1': 'VGPRField', 'VDATA': 'VGPRField',
+  'VADDR': 'VGPRField', 'ADDR': 'VGPRField', 'DATA': 'VGPRField', 'DATA0': 'VGPRField', 'DATA1': 'VGPRField', 'SIMM16': 'SImm', 'OFFSET': 'Imm',
+  'OPX': 'VOPDOp', 'OPY': 'VOPDOp', 'SRCX0': 'Src', 'SRCY0': 'Src', 'VSRCX1': 'VGPRField', 'VSRCY1': 'VGPRField', 'VDSTX': 'VGPRField', 'VDSTY': 'VDSTYEnc'}
+FIELD_ORDER = {
+  'SOP2': ['op', 'sdst', 'ssrc0', 'ssrc1'], 'SOP1': ['op', 'sdst', 'ssrc0'], 'SOPC': ['op', 'ssrc0', 'ssrc1'],
+  'SOPK': ['op', 'sdst', 'simm16'], 'SOPP': ['op', 'simm16'], 'VOP1': ['op', 'vdst', 'src0'], 'VOPC': ['op', 'src0', 'vsrc1'],
+  'VOP2': ['op', 'vdst', 'src0', 'vsrc1'], 'VOP3SD': ['op', 'vdst', 'sdst', 'src0', 'src1', 'src2', 'clmp'],
+  'SMEM': ['op', 'sdata', 'sbase', 'soffset', 'offset', 'glc', 'dlc'], 'DS': ['op', 'vdst', 'addr', 'data0', 'data1'],
+  'VOP3': ['op', 'vdst', 'src0', 'src1', 'src2', 'omod', 'neg', 'abs', 'clmp', 'opsel'],
+  'VOP3P': ['op', 'vdst', 'src0', 'src1', 'src2', 'neg', 'neg_hi', 'opsel', 'opsel_hi', 'clmp'],
+  'FLAT': ['op', 'vdst', 'addr', 'data', 'saddr', 'offset', 'seg', 'dlc', 'glc', 'slc'],
+  'MUBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
+  'MTBUF': ['op', 'vdata', 'vaddr', 'srsrc', 'soffset', 'offset', 'format', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe'],
+  'MIMG': ['op', 'vdata', 'vaddr', 'srsrc', 'ssamp', 'dmask', 'dim', 'unrm', 'dlc', 'glc', 'slc'],
+  'EXP': ['en', 'target', 'vsrc0', 'vsrc1', 'vsrc2', 'vsrc3', 'done', 'row'],
+  'VINTERP': ['op', 'vdst', 'src0', 'src1', 'src2', 'waitexp', 'clmp', 'opsel', 'neg'],
+  'VOPD': ['opx', 'opy', 'vdstx', 'vdsty', 'srcx0', 'vsrcx1', 'srcy0', 'vsrcy1'],
+  'LDSDIR': ['op', 'vdst', 'attr', 'attr_chan', 'wait_va']}
+SRC_EXTRAS = {233: 'DPP8', 234: 'DPP8FI', 250: 'DPP16', 251: 'VCCZ', 252: 'EXECZ', 254: 'LDS_DIRECT'}
+FLOAT_MAP = {'0.5': 'POS_HALF', '-0.5': 'NEG_HALF', '1.0': 'POS_ONE', '-1.0': 'NEG_ONE', '2.0': 'POS_TWO', '-2.0': 'NEG_TWO',
+  '4.0': 'POS_FOUR', '-4.0': 'NEG_FOUR', '1/(2*PI)': 'INV_2PI', '0': 'ZERO'}
+
+def _parse_bits(s: str) -> tuple[int, int] | None:
+  import re
+  return (int(m.group(1)), int(m.group(2) or m.group(1))) if (m := re.match(r'\[(\d+)(?::(\d+))?\]', s)) else None
+
+def _parse_fields_table(table: list, fmt: str, enums: set[str]) -> list[tuple]:
+  import re
+  fields = []
+  for row in table[1:]:
+    if not row or not row[0]: continue
+    name, bits_str = row[0].split('\n')[0].strip(), (row[1] or '').split('\n')[0].strip()
+    if not (bits := _parse_bits(bits_str)): continue
+    enc_val, hi, lo = None, bits[0], bits[1]
+    if name == 'ENCODING' and row[2]:
+      # Handle both RDNA3 ('bXX) and CDNA4 (Must be: XX) encoding formats
+      if m := re.search(r"(?:'b|Must be:\s*)([01_]+)", row[2]):
+        enc_bits = m.group(1).replace('_', '')
+        enc_val = int(enc_bits, 2)
+        declared_width, actual_width = hi - lo + 1, len(enc_bits)
+        if actual_width > declared_width: lo = hi - actual_width + 1
+    ftype = f"{fmt}Op" if name == 'OP' and f"{fmt}Op" in enums else FIELD_TYPES.get(name.upper())
+    fields.append((name, hi, lo, enc_val, ftype))
+  return fields
+
+def _parse_single_pdf(url: str) -> dict:
+  """Parse a single PDF and return raw data (formats, enums, src_enum, doc_name, is_cdna)."""
+  import re, pdfplumber
+  from tinygrad.helpers import fetch
+
+  pdf = pdfplumber.open(fetch(url))
+
+  # Auto-detect document type from first page
+  first_page_text = pdf.pages[0].extract_text() or ''
+  is_cdna4 = 'CDNA4' in first_page_text or 'CDNA 4' in first_page_text
+  is_cdna3 = 'CDNA3' in first_page_text or 'CDNA 3' in first_page_text or 'MI300' in first_page_text
+  is_cdna = is_cdna3 or is_cdna4
+  is_rdna4 = 'RDNA4' in first_page_text or 'RDNA 4' in first_page_text
+  is_rdna35 = 'RDNA3.5' in first_page_text or 'RDNA 3.5' in first_page_text  # Check 3.5 before 3
+  is_rdna3 = not is_rdna35 and ('RDNA3' in first_page_text or 'RDNA 3' in first_page_text)
+  doc_name = "CDNA4" if is_cdna4 else "CDNA3" if is_cdna3 else "RDNA4" if is_rdna4 else "RDNA3.5" if is_rdna35 else "RDNA3" if is_rdna3 else "Unknown"
+
+  # Find the "Microcode Formats" section - search for SOP2 format definition
+  microcode_start = None
+  total_pages = len(pdf.pages)
+  # Search from likely locations (formats are typically 20-95% through the document - RDNA3 has them at ~25%)
+  for i in range(int(total_pages * 0.2), total_pages):
+    text = pdf.pages[i].extract_text() or ''
+    # Look for "X.Y.Z. SOP2" section header or "Chapter X. Microcode Formats"
+    if re.search(r'\d+\.\d+\.\d+\.\s+SOP2\b', text) or re.search(r'Chapter \d+\.\s+Microcode Formats', text):
+      microcode_start = i
+      break
+  if microcode_start is None: microcode_start = int(total_pages * 0.9)
+
+  pages = pdf.pages[microcode_start:microcode_start + 50]
+  page_texts = [p.extract_text() or '' for p in pages]
+  page_tables = [[t.extract() for t in p.find_tables()] for p in pages]
+  full_text = '\n'.join(page_texts)
+
+  # parse SSRC encoding from first page with VCC_LO
+  src_enum = dict(SRC_EXTRAS)
+  for text in page_texts[:10]:
+    if 'SSRC0' in text and 'VCC_LO' in text:
+      for m in re.finditer(r'^(\d+)\s+(\S+)', text, re.M):
+        val, name = int(m.group(1)), m.group(2).rstrip('.:')
+        if name in FLOAT_MAP: src_enum[val] = FLOAT_MAP[name]
+        elif re.match(r'^[A-Z][A-Z0-9_]*$', name): src_enum[val] = name
+      break
+
+  # parse opcode tables
+  enums: dict[str, dict[int, str]] = {}
+  for m in re.finditer(r'Table \d+\. (\w+) Opcodes(.*?)(?=Table \d+\.|\n\d+\.\d+\.\d+\.\s+\w+\s*\nDescription|$)', full_text, re.S):
+    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+([A-Z][A-Z0-9_]+)', m.group(2))}:
+      enums[m.group(1) + "Op"] = ops
+  if vopd_m := re.search(r'Table \d+\. VOPD Y-Opcodes\n(.*?)(?=Table \d+\.|15\.\d)', full_text, re.S):
+    if ops := {int(x.group(1)): x.group(2) for x in re.finditer(r'(\d+)\s+(V_DUAL_\w+)', vopd_m.group(1))}:
+      enums["VOPDOp"] = ops
+  enum_names = set(enums.keys())
+
+  def is_fields_table(t) -> bool: return t and len(t) > 1 and t[0] and 'Field' in str(t[0][0] or '')
+  def has_encoding(fields) -> bool: return any(f[0] == 'ENCODING' for f in fields)
+  def has_header_before_fields(text) -> bool:
+    return (pos := text.find('Field Name')) != -1 and bool(re.search(r'\d+\.\d+\.\d+\.\s+\w+\s*\n', text[:pos]))
+
+  # find format headers with their page indices
+  format_headers = []
+  for i, text in enumerate(page_texts):
+    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n?Description', text): format_headers.append((m.group(1), i, m.start()))
+    for m in re.finditer(r'\d+\.\d+\.\d+\.\s+(\w+)\s*\n', text):
+      fmt_name = m.group(1)
+      if is_cdna and fmt_name.isupper() and len(fmt_name) >= 2:
+        format_headers.append((fmt_name, i, m.start()))
+      elif m.start() > len(text) - 200 and 'Description' not in text[m.end():] and i + 1 < len(page_texts):
+        next_text = page_texts[i + 1].lstrip()
+        if next_text.startswith('Description') or (next_text.startswith('"RDNA') and 'Description' in next_text[:200]):
+          format_headers.append((fmt_name, i, m.start()))
+
+  # parse instruction formats
+  formats: dict[str, list] = {}
+  for fmt_name, page_idx, header_pos in format_headers:
+    if fmt_name in formats: continue
+    text, tables = page_texts[page_idx], page_tables[page_idx]
+    field_pos = text.find('Field Name', header_pos)
+
+    fields = None
+    for offset in range(3):
+      if page_idx + offset >= len(pages): break
+      if offset > 0 and has_header_before_fields(page_texts[page_idx + offset]): break
+      for t in page_tables[page_idx + offset] if offset > 0 or field_pos > header_pos else []:
+        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)) and has_encoding(f):
+          fields = f
+          break
+      if fields: break
+
+    if not fields and field_pos > header_pos:
+      for t in tables:
+        if is_fields_table(t) and (f := _parse_fields_table(t, fmt_name, enum_names)):
+          fields = f
+          break
+
+    if not fields: continue
+    field_names = {f[0] for f in fields}
+
+    for pg_offset in range(1, 3):
+      if page_idx + pg_offset >= len(pages) or has_header_before_fields(page_texts[page_idx + pg_offset]): break
+      for t in page_tables[page_idx + pg_offset]:
+        if is_fields_table(t) and (extra := _parse_fields_table(t, fmt_name, enum_names)) and not has_encoding(extra):
+          for ef in extra:
+            if ef[0] not in field_names:
+              fields.append(ef)
+              field_names.add(ef[0])
+          break
+    formats[fmt_name] = fields
+
+  # fix known PDF errors
+  if 'SMEM' in formats:
+    formats['SMEM'] = [(n, 13 if n == 'DLC' else 14 if n == 'GLC' else h, 13 if n == 'DLC' else 14 if n == 'GLC' else l, e, t)
+                       for n, h, l, e, t in formats['SMEM']]
+
+  return {"formats": formats, "enums": enums, "src_enum": src_enum, "doc_name": doc_name, "is_cdna": is_cdna}
+
+def _merge_results(results: list[dict]) -> dict:
+  """Merge multiple PDF parse results into a superset. Asserts if any conflicts."""
+  merged = {"formats": {}, "enums": {}, "src_enum": dict(SRC_EXTRAS), "doc_names": [], "is_cdna": False}
+  for r in results:
+    merged["doc_names"].append(r["doc_name"])
+    merged["is_cdna"] = merged["is_cdna"] or r["is_cdna"]
+    # Merge src_enum (union, assert no conflicts)
+    for val, name in r["src_enum"].items():
+      if val in merged["src_enum"]:
+        assert merged["src_enum"][val] == name, f"SrcEnum conflict: {val} = {merged['src_enum'][val]} vs {name}"
+      else:
+        merged["src_enum"][val] = name
+    # Merge enums (union of ops per enum, assert no conflicts)
+    for enum_name, ops in r["enums"].items():
+      if enum_name not in merged["enums"]: merged["enums"][enum_name] = {}
+      for val, name in ops.items():
+        if val in merged["enums"][enum_name]:
+          assert merged["enums"][enum_name][val] == name, f"{enum_name} conflict: {val} = {merged['enums'][enum_name][val]} vs {name}"
+        else:
+          merged["enums"][enum_name][val] = name
+    # Merge formats (union of fields, assert no bit position conflicts for same field name)
+    for fmt_name, fields in r["formats"].items():
+      if fmt_name not in merged["formats"]:
+        merged["formats"][fmt_name] = list(fields)
+      else:
+        existing = {f[0]: (f[1], f[2]) for f in merged["formats"][fmt_name]}  # name -> (hi, lo)
+        for f in fields:
+          name, hi, lo = f[0], f[1], f[2]
+          if name in existing:
+            assert existing[name] == (hi, lo), f"Format {fmt_name} field {name} conflict: bits {existing[name]} vs ({hi}, {lo})"
+          else:
+            merged["formats"][fmt_name].append(f)
+  return merged
+
+def generate(output_path: str | None = None, arch: str = "rdna3") -> dict:
+  """Generate instruction definitions from AMD ISA PDF(s). Returns dict with formats for testing."""
+  urls = PDF_URLS[arch]
+  if isinstance(urls, str): urls = [urls]
+
+  # Parse all PDFs and merge
+  results = [_parse_single_pdf(url) for url in urls]
+  if len(results) == 1:
+    merged = results[0]
+    doc_name = merged["doc_name"]
+  else:
+    merged = _merge_results(results)
+    doc_name = "+".join(merged["doc_names"])
+
+  formats, enums, src_enum = merged["formats"], merged["enums"], merged["src_enum"]
+
+  # generate output
+  def enum_lines(name, items):
+    return [f"class {name}(IntEnum):"] + [f"  {n} = {v}" for v, n in sorted(items.items())] + [""]
+  def field_key(f): return order.index(f[0].lower()) if f[0].lower() in order else 1000
+  lines = [f"# autogenerated from AMD {doc_name} ISA PDF by dsl.py - do not edit", "from enum import IntEnum",
+           "from typing import Annotated",
+           "from extra.assembly.amd.dsl import bits, BitField, Inst32, Inst64, SGPR, VGPR, TTMP as TTMP, s as s, v as v, ttmp as ttmp, SSrc, Src, SImm, Imm, VDSTYEnc, SGPRField, VGPRField",
+           "import functools", ""]
+  lines += enum_lines("SrcEnum", src_enum) + sum([enum_lines(n, ops) for n, ops in sorted(enums.items())], [])
+  # Format-specific field defaults (verified against LLVM test vectors)
+  format_defaults = {'VOP3P': {'opsel_hi': 3, 'opsel_hi2': 1}}
+  lines.append("# instruction formats")
+  for fmt_name, fields in sorted(formats.items()):
+    base = "Inst64" if max(f[1] for f in fields) > 31 or fmt_name == 'VOP3SD' else "Inst32"
+    order = FIELD_ORDER.get(fmt_name, [])
+    lines.append(f"class {fmt_name}({base}):")
+    if enc := next((f for f in fields if f[0] == 'ENCODING'), None):
+      enc_str = f"bits[{enc[1]}:{enc[2]}] == 0b{enc[3]:b}" if enc[1] != enc[2] else f"bits[{enc[1]}] == {enc[3]}"
+      lines.append(f"  encoding = {enc_str}")
+    if defaults := format_defaults.get(fmt_name):
+      lines.append(f"  _defaults = {defaults}")
+    for name, hi, lo, _, ftype in sorted([f for f in fields if f[0] != 'ENCODING'], key=field_key):
+      if ftype and ftype.endswith('Op'):
+        ann = f":Annotated[BitField, {ftype}]"
+      else:
+        ann = f":{ftype}" if ftype else ""
+      lines.append(f"  {name.lower()}{ann} = bits[{hi}]" if hi == lo else f"  {name.lower()}{ann} = bits[{hi}:{lo}]")
+    lines.append("")
+  lines.append("# instruction helpers")
+  for cls_name, ops in sorted(enums.items()):
+    fmt = cls_name[:-2]
+    for op_val, name in sorted(ops.items()):
+      seg = {"GLOBAL": ", seg=2", "SCRATCH": ", seg=2"}.get(fmt, "")
+      tgt = {"GLOBAL": "FLAT, GLOBALOp", "SCRATCH": "FLAT, SCRATCHOp"}.get(fmt, f"{fmt}, {cls_name}")
+      if fmt in formats or fmt in ("GLOBAL", "SCRATCH"):
+        if fmt in ("VOP1", "VOP2", "VOPC"):
+          suffix = "_e32"
+        elif fmt == "VOP3" and op_val < 512:
+          suffix = "_e64"
+        else:
+          suffix = ""
+        if name in ('V_FMAMK_F32', 'V_FMAMK_F16'):
+          lines.append(f"def {name.lower()}{suffix}(vdst, src0, K, vsrc1): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
+        elif name in ('V_FMAAK_F32', 'V_FMAAK_F16'):
+          lines.append(f"def {name.lower()}{suffix}(vdst, src0, vsrc1, K): return {fmt}({cls_name}.{name}, vdst, src0, vsrc1, literal=K)")
+        else:
+          lines.append(f"{name.lower()}{suffix} = functools.partial({tgt}.{name}{seg})")
+  skip_exports = {'DPP8', 'DPP16'}
+  src_names = {name for _, name in src_enum.items()}
+  lines += [""] + [f"{name} = SrcEnum.{name}" for _, name in sorted(src_enum.items()) if name not in skip_exports]
+  if "NULL" in src_names: lines.append("OFF = NULL\n")
+
+  if output_path is not None:
+    import pathlib
+    pathlib.Path(output_path).write_text('\n'.join(lines))
+  return {"formats": formats, "enums": enums, "src_enum": src_enum}
+
+if __name__ == "__main__":
+  import argparse
+  parser = argparse.ArgumentParser(description="Generate instruction definitions from AMD ISA PDF")
+  parser.add_argument("--arch", choices=list(PDF_URLS.keys()) + ["all"], default="rdna3", help="Target architecture (default: rdna3)")
+  args = parser.parse_args()
+  if args.arch == "all":
+    for arch in PDF_URLS.keys():
+      result = generate(f"extra/assembly/amd/autogen/{arch}/__init__.py", arch=arch)
+      print(f"{arch}: generated SrcEnum ({len(result['src_enum'])}) + {len(result['enums'])} opcode enums + {len(result['formats'])} format classes")
+  else:
+    result = generate(f"extra/assembly/amd/autogen/{args.arch}/__init__.py", arch=args.arch)
+    print(f"generated SrcEnum ({len(result['src_enum'])}) + {len(result['enums'])} opcode enums + {len(result['formats'])} format classes")

extra/assembly/amd/emu.py

@@ -0,0 +1,759 @@
+# RDNA3 emulator - executes compiled pseudocode from AMD ISA PDF
+# mypy: ignore-errors
+from __future__ import annotations
+import ctypes, os
+from extra.assembly.amd.dsl import Inst, RawImm
+from extra.assembly.amd.pcode import _f32, _i32, _sext, _f16, _i16, _f64, _i64, Reg
+from extra.assembly.amd.autogen.rdna3.gen_pcode import get_compiled_functions
+from extra.assembly.amd.autogen.rdna3 import (
+  SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, VOP1, VOP2, VOP3, VOP3SD, VOP3P, VOPC, DS, FLAT, VOPD, SrcEnum,
+  SOP1Op, SOP2Op, SOPCOp, SOPKOp, SOPPOp, SMEMOp, VOP1Op, VOP2Op, VOP3Op, VOP3SDOp, VOP3POp, VOPCOp, DSOp, FLATOp, GLOBALOp, VOPDOp
+)
+
+Program = dict[int, Inst]
+WAVE_SIZE, SGPR_COUNT, VGPR_COUNT = 32, 128, 256
+VCC_LO, VCC_HI, NULL, EXEC_LO, EXEC_HI, SCC = SrcEnum.VCC_LO, SrcEnum.VCC_HI, SrcEnum.NULL, SrcEnum.EXEC_LO, SrcEnum.EXEC_HI, SrcEnum.SCC
+
+# VOP3 ops that use 64-bit operands (and thus 64-bit literals when src is 255)
+# Exception: V_LDEXP_F64 has 32-bit integer src1, so literal should NOT be 64-bit when src1=255
+_VOP3_64BIT_OPS = {op.value for op in VOP3Op if op.name.endswith(('_F64', '_B64', '_I64', '_U64'))}
+# Ops where src1 is 32-bit (exponent/shift amount) even though the op name suggests 64-bit
+_VOP3_64BIT_OPS_32BIT_SRC1 = {VOP3Op.V_LDEXP_F64.value}
+# Ops with 16-bit types in name (for source/dest handling)
+# Exception: SAD/MSAD ops take 32-bit packed sources and extract 16-bit/8-bit chunks internally
+_VOP3_16BIT_OPS = {op for op in VOP3Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16')) and 'SAD' not in op.name}
+_VOP1_16BIT_OPS = {op for op in VOP1Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
+_VOP2_16BIT_OPS = {op for op in VOP2Op if any(s in op.name for s in ('_F16', '_B16', '_I16', '_U16'))}
+# CVT ops with 32/64-bit source (despite 16-bit in name)
+_CVT_32_64_SRC_OPS = {op for op in VOP3Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))} | \
+                     {op for op in VOP1Op if op.name.startswith('V_CVT_') and op.name.endswith(('_F32', '_I32', '_U32', '_F64', '_I64', '_U64'))}
+# 16-bit dst ops (PACK has 32-bit dst despite F16 in name)
+_VOP3_16BIT_DST_OPS = {op for op in _VOP3_16BIT_OPS if 'PACK' not in op.name}
+_VOP1_16BIT_DST_OPS = {op for op in _VOP1_16BIT_OPS if 'PACK' not in op.name}
+
+# Inline constants for src operands 128-254. Build tables for f32, f16, and f64 formats.
+import struct as _struct
+_FLOAT_CONSTS = {SrcEnum.POS_HALF: 0.5, SrcEnum.NEG_HALF: -0.5, SrcEnum.POS_ONE: 1.0, SrcEnum.NEG_ONE: -1.0,
+                 SrcEnum.POS_TWO: 2.0, SrcEnum.NEG_TWO: -2.0, SrcEnum.POS_FOUR: 4.0, SrcEnum.NEG_FOUR: -4.0, SrcEnum.INV_2PI: 0.15915494309189535}
+def _build_inline_consts(neg_mask, float_to_bits):
+  tbl = list(range(65)) + [((-i) & neg_mask) for i in range(1, 17)] + [0] * (127 - 81)
+  for k, v in _FLOAT_CONSTS.items(): tbl[k - 128] = float_to_bits(v)
+  return tbl
+_INLINE_CONSTS = _build_inline_consts(0xffffffff, lambda f: _struct.unpack('<I', _struct.pack('<f', f))[0])
+_INLINE_CONSTS_F16 = _build_inline_consts(0xffff, lambda f: _struct.unpack('<H', _struct.pack('<e', f))[0])
+_INLINE_CONSTS_F64 = _build_inline_consts(0xffffffffffffffff, lambda f: _struct.unpack('<Q', _struct.pack('<d', f))[0])
+
+# Memory access
+_valid_mem_ranges: list[tuple[int, int]] = []
+def set_valid_mem_ranges(ranges: set[tuple[int, int]]) -> None: _valid_mem_ranges.clear(); _valid_mem_ranges.extend(ranges)
+def _mem_valid(addr: int, size: int) -> bool:
+  for s, z in _valid_mem_ranges:
+    if s <= addr and addr + size <= s + z: return True
+  return not _valid_mem_ranges
+def _ctypes_at(addr: int, size: int): return (ctypes.c_uint8 if size == 1 else ctypes.c_uint16 if size == 2 else ctypes.c_uint32).from_address(addr)
+def mem_read(addr: int, size: int) -> int: return _ctypes_at(addr, size).value if _mem_valid(addr, size) else 0
+def mem_write(addr: int, size: int, val: int) -> None:
+  if _mem_valid(addr, size): _ctypes_at(addr, size).value = val
+
+# Memory op tables (not pseudocode - these are format descriptions)
+def _mem_ops(ops, suffix_map):
+  return {getattr(e, f"{p}_{s}"): v for e in ops for s, v in suffix_map.items() for p in [e.__name__.replace("Op", "")]}
+_LOAD_MAP = {'LOAD_B32': (1,4,0), 'LOAD_B64': (2,4,0), 'LOAD_B96': (3,4,0), 'LOAD_B128': (4,4,0), 'LOAD_U8': (1,1,0), 'LOAD_I8': (1,1,1), 'LOAD_U16': (1,2,0), 'LOAD_I16': (1,2,1)}
+_STORE_MAP = {'STORE_B32': (1,4), 'STORE_B64': (2,4), 'STORE_B96': (3,4), 'STORE_B128': (4,4), 'STORE_B8': (1,1), 'STORE_B16': (1,2)}
+FLAT_LOAD, FLAT_STORE = _mem_ops([GLOBALOp, FLATOp], _LOAD_MAP), _mem_ops([GLOBALOp, FLATOp], _STORE_MAP)
+# D16 ops: load/store 16-bit to lower or upper half of VGPR. Format: (size, sign, hi) where hi=1 means upper 16 bits
+_D16_LOAD_MAP = {'LOAD_D16_U8': (1,0,0), 'LOAD_D16_I8': (1,1,0), 'LOAD_D16_B16': (2,0,0),
+                 'LOAD_D16_HI_U8': (1,0,1), 'LOAD_D16_HI_I8': (1,1,1), 'LOAD_D16_HI_B16': (2,0,1)}
+_D16_STORE_MAP = {'STORE_D16_HI_B8': (1,1), 'STORE_D16_HI_B16': (2,1)}  # (size, hi)
+FLAT_D16_LOAD = _mem_ops([GLOBALOp, FLATOp], _D16_LOAD_MAP)
+FLAT_D16_STORE = _mem_ops([GLOBALOp, FLATOp], _D16_STORE_MAP)
+DS_LOAD = {DSOp.DS_LOAD_B32: (1,4,0), DSOp.DS_LOAD_B64: (2,4,0), DSOp.DS_LOAD_B128: (4,4,0), DSOp.DS_LOAD_U8: (1,1,0), DSOp.DS_LOAD_I8: (1,1,1), DSOp.DS_LOAD_U16: (1,2,0), DSOp.DS_LOAD_I16: (1,2,1)}
+DS_STORE = {DSOp.DS_STORE_B32: (1,4), DSOp.DS_STORE_B64: (2,4), DSOp.DS_STORE_B128: (4,4), DSOp.DS_STORE_B8: (1,1), DSOp.DS_STORE_B16: (1,2)}
+SMEM_LOAD = {SMEMOp.S_LOAD_B32: 1, SMEMOp.S_LOAD_B64: 2, SMEMOp.S_LOAD_B128: 4, SMEMOp.S_LOAD_B256: 8, SMEMOp.S_LOAD_B512: 16}
+
+# VOPD op -> VOP3 op mapping (VOPD is dual-issue of VOP1/VOP2 ops, use VOP3 enums for pseudocode lookup)
+_VOPD_TO_VOP = {
+  VOPDOp.V_DUAL_FMAC_F32: VOP3Op.V_FMAC_F32, VOPDOp.V_DUAL_FMAAK_F32: VOP2Op.V_FMAAK_F32, VOPDOp.V_DUAL_FMAMK_F32: VOP2Op.V_FMAMK_F32,
+  VOPDOp.V_DUAL_MUL_F32: VOP3Op.V_MUL_F32, VOPDOp.V_DUAL_ADD_F32: VOP3Op.V_ADD_F32, VOPDOp.V_DUAL_SUB_F32: VOP3Op.V_SUB_F32,
+  VOPDOp.V_DUAL_SUBREV_F32: VOP3Op.V_SUBREV_F32, VOPDOp.V_DUAL_MUL_DX9_ZERO_F32: VOP3Op.V_MUL_DX9_ZERO_F32,
+  VOPDOp.V_DUAL_MOV_B32: VOP3Op.V_MOV_B32, VOPDOp.V_DUAL_CNDMASK_B32: VOP3Op.V_CNDMASK_B32,
+  VOPDOp.V_DUAL_MAX_F32: VOP3Op.V_MAX_F32, VOPDOp.V_DUAL_MIN_F32: VOP3Op.V_MIN_F32,
+  VOPDOp.V_DUAL_ADD_NC_U32: VOP3Op.V_ADD_NC_U32, VOPDOp.V_DUAL_LSHLREV_B32: VOP3Op.V_LSHLREV_B32, VOPDOp.V_DUAL_AND_B32: VOP3Op.V_AND_B32,
+}
+
+# Compiled pseudocode functions (lazy loaded)
+_COMPILED: dict | None = None
+
+def _get_compiled() -> dict:
+  global _COMPILED
+  if _COMPILED is None: _COMPILED = get_compiled_functions()
+  return _COMPILED
+
+class WaveState:
+  __slots__ = ('sgpr', 'vgpr', 'scc', 'pc', 'literal', '_pend_sgpr', '_scc_reg', '_vcc_reg', '_exec_reg')
+  def __init__(self):
+    self.sgpr = [Reg(0) for _ in range(SGPR_COUNT)]
+    self.vgpr = [[Reg(0) for _ in range(VGPR_COUNT)] for _ in range(WAVE_SIZE)]
+    self.sgpr[EXEC_LO]._val = 0xffffffff
+    self.scc, self.pc, self.literal, self._pend_sgpr = 0, 0, 0, {}
+    # Reg wrappers for pseudocode access
+    self._scc_reg = Reg(0)
+    self._vcc_reg = self.sgpr[VCC_LO]
+    self._exec_reg = self.sgpr[EXEC_LO]
+
+  @property
+  def vcc(self) -> int: return self.sgpr[VCC_LO]._val | (self.sgpr[VCC_HI]._val << 32)
+  @vcc.setter
+  def vcc(self, v: int): self.sgpr[VCC_LO]._val, self.sgpr[VCC_HI]._val = v & 0xffffffff, (v >> 32) & 0xffffffff
+  @property
+  def exec_mask(self) -> int: return self.sgpr[EXEC_LO]._val | (self.sgpr[EXEC_HI]._val << 32)
+  @exec_mask.setter
+  def exec_mask(self, v: int): self.sgpr[EXEC_LO]._val, self.sgpr[EXEC_HI]._val = v & 0xffffffff, (v >> 32) & 0xffffffff
+
+  def rsgpr(self, i: int) -> int: return 0 if i == NULL else self.scc if i == SCC else self.sgpr[i]._val if i < SGPR_COUNT else 0
+  def wsgpr(self, i: int, v: int):
+    if i < SGPR_COUNT and i != NULL: self.sgpr[i]._val = v & 0xffffffff
+  def rsgpr64(self, i: int) -> int: return self.rsgpr(i) | (self.rsgpr(i+1) << 32)
+  def wsgpr64(self, i: int, v: int): self.wsgpr(i, v & 0xffffffff); self.wsgpr(i+1, (v >> 32) & 0xffffffff)
+
+  def rsrc(self, v: int, lane: int) -> int:
+    if v < SGPR_COUNT: return self.sgpr[v]._val
+    if v == SCC: return self.scc
+    if v < 255: return _INLINE_CONSTS[v - 128]
+    if v == 255: return self.literal
+    return self.vgpr[lane][v - 256]._val if v <= 511 else 0
+
+  def rsrc_reg(self, v: int, lane: int) -> Reg:
+    """Return the Reg object for a source operand."""
+    if v < SGPR_COUNT: return self.sgpr[v]
+    if v == SCC: self._scc_reg._val = self.scc; return self._scc_reg
+    if v < 255: return Reg(_INLINE_CONSTS[v - 128])
+    if v == 255: return Reg(self.literal)
+    return self.vgpr[lane][v - 256] if v <= 511 else Reg(0)
+
+  def rsrc_f16(self, v: int, lane: int) -> int:
+    """Read source operand for VOP3P packed f16 operations. Uses f16 inline constants."""
+    if v < SGPR_COUNT: return self.sgpr[v]._val
+    if v == SCC: return self.scc
+    if v < 255: return _INLINE_CONSTS_F16[v - 128]
+    if v == 255: return self.literal
+    return self.vgpr[lane][v - 256]._val if v <= 511 else 0
+
+  def rsrc_reg_f16(self, v: int, lane: int) -> Reg:
+    """Return Reg for VOP3P source. Inline constants are f16 in low 16 bits only."""
+    if v < SGPR_COUNT: return self.sgpr[v]
+    if v == SCC: self._scc_reg._val = self.scc; return self._scc_reg
+    if v < 255: return Reg(_INLINE_CONSTS_F16[v - 128])  # f16 inline constant
+    if v == 255: return Reg(self.literal)
+    return self.vgpr[lane][v - 256] if v <= 511 else Reg(0)
+
+  def rsrc64(self, v: int, lane: int) -> int:
+    """Read 64-bit source operand. For inline constants, returns 64-bit representation."""
+    if 128 <= v < 255: return _INLINE_CONSTS_F64[v - 128]
+    if v == 255: return self.literal
+    return self.rsrc(v, lane) | ((self.rsrc(v+1, lane) if v < VCC_LO or 256 <= v <= 511 else 0) << 32)
+
+  def rsrc_reg64(self, v: int, lane: int) -> Reg:
+    """Return Reg for 64-bit source operand. For inline constants, returns 64-bit f64 value."""
+    if 128 <= v < 255: return Reg(_INLINE_CONSTS_F64[v - 128])
+    if v == 255: return Reg(self.literal)
+    if v < SGPR_COUNT: return Reg(self.sgpr[v]._val | (self.sgpr[v+1]._val << 32))
+    if 256 <= v <= 511:
+      vgpr_idx = v - 256
+      return Reg(self.vgpr[lane][vgpr_idx]._val | (self.vgpr[lane][vgpr_idx + 1]._val << 32))
+    return Reg(0)
+
+  def pend_sgpr_lane(self, reg: int, lane: int, val: int):
+    if reg not in self._pend_sgpr: self._pend_sgpr[reg] = 0
+    if val: self._pend_sgpr[reg] |= (1 << lane)
+  def commit_pends(self):
+    for reg, val in self._pend_sgpr.items(): self.sgpr[reg]._val = val
+    self._pend_sgpr.clear()
+
+# Instruction decode
+def decode_format(word: int) -> tuple[type[Inst] | None, bool]:
+  hi2 = (word >> 30) & 0x3
+  if hi2 == 0b11:
+    enc = (word >> 26) & 0xf
+    if enc == 0b1101: return SMEM, True
+    if enc == 0b0101:
+      op = (word >> 16) & 0x3ff
+      return (VOP3SD, True) if op in (288, 289, 290, 764, 765, 766, 767, 768, 769, 770) else (VOP3, True)
+    return {0b0011: (VOP3P, True), 0b0110: (DS, True), 0b0111: (FLAT, True), 0b0010: (VOPD, True)}.get(enc, (None, True))
+  if hi2 == 0b10:
+    enc = (word >> 23) & 0x7f
+    return {0b1111101: (SOP1, False), 0b1111110: (SOPC, False), 0b1111111: (SOPP, False)}.get(enc, (SOPK, False) if ((word >> 28) & 0xf) == 0b1011 else (SOP2, False))
+  enc = (word >> 25) & 0x7f
+  return (VOPC, False) if enc == 0b0111110 else (VOP1, False) if enc == 0b0111111 else (VOP2, False)
+
+def _unwrap(v) -> int: return v.val if isinstance(v, RawImm) else v.value if hasattr(v, 'value') else v
+
+def decode_program(data: bytes) -> Program:
+  result: Program = {}
+  i = 0
+  while i < len(data):
+    word = int.from_bytes(data[i:i+4], 'little')
+    inst_class, is_64 = decode_format(word)
+    if inst_class is None: i += 4; continue
+    base_size = 8 if is_64 else 4
+    # Pass enough data for potential 64-bit literal (base + 8 bytes max)
+    inst = inst_class.from_bytes(data[i:i+base_size+8])
+    for name, val in inst._values.items(): setattr(inst, name, _unwrap(val))
+    # from_bytes already handles literal reading - only need fallback for cases it doesn't handle
+    if inst._literal is None:
+      has_literal = any(getattr(inst, fld, None) == 255 for fld in ('src0', 'src1', 'src2', 'ssrc0', 'ssrc1', 'srcx0', 'srcy0'))
+      if inst_class == VOP2 and inst.op in (44, 45, 55, 56): has_literal = True
+      if inst_class == VOPD and (inst.opx in (1, 2) or inst.opy in (1, 2)): has_literal = True
+      if inst_class == SOP2 and inst.op in (69, 70): has_literal = True
+      if has_literal:
+        # For 64-bit ops, the 32-bit literal is placed in HIGH 32 bits (low 32 bits = 0)
+        # Exception: some ops have mixed src sizes (e.g., V_LDEXP_F64 has 32-bit src1)
+        op_val = inst._values.get('op')
+        if hasattr(op_val, 'value'): op_val = op_val.value
+        is_64bit = inst_class is VOP3 and op_val in _VOP3_64BIT_OPS
+        # Don't treat literal as 64-bit if the op has 32-bit src1 and src1 is the literal
+        if is_64bit and op_val in _VOP3_64BIT_OPS_32BIT_SRC1 and getattr(inst, 'src1', None) == 255:
+          is_64bit = False
+        lit32 = int.from_bytes(data[i+base_size:i+base_size+4], 'little')
+        inst._literal = (lit32 << 32) if is_64bit else lit32
+    inst._words = inst.size() // 4
+    result[i // 4] = inst
+    i += inst._words * 4
+  return result
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# EXECUTION - All ALU ops use pseudocode from PDF
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def exec_scalar(st: WaveState, inst: Inst) -> int:
+  """Execute scalar instruction. Returns PC delta or negative for special cases."""
+  compiled = _get_compiled()
+  inst_type = type(inst)
+
+  # SOPP: control flow (not ALU)
+  if inst_type is SOPP:
+    op = inst.op
+    if op == SOPPOp.S_ENDPGM: return -1
+    if op == SOPPOp.S_BARRIER: return -2
+    if op == SOPPOp.S_BRANCH: return _sext(inst.simm16, 16)
+    if op == SOPPOp.S_CBRANCH_SCC0: return _sext(inst.simm16, 16) if st.scc == 0 else 0
+    if op == SOPPOp.S_CBRANCH_SCC1: return _sext(inst.simm16, 16) if st.scc == 1 else 0
+    if op == SOPPOp.S_CBRANCH_VCCZ: return _sext(inst.simm16, 16) if (st.vcc & 0xffffffff) == 0 else 0
+    if op == SOPPOp.S_CBRANCH_VCCNZ: return _sext(inst.simm16, 16) if (st.vcc & 0xffffffff) != 0 else 0
+    if op == SOPPOp.S_CBRANCH_EXECZ: return _sext(inst.simm16, 16) if st.exec_mask == 0 else 0
+    if op == SOPPOp.S_CBRANCH_EXECNZ: return _sext(inst.simm16, 16) if st.exec_mask != 0 else 0
+    # Valid SOPP range is 0-61 (max defined opcode); anything above is invalid
+    if op > 61: raise NotImplementedError(f"Invalid SOPP opcode {op}")
+    return 0  # waits, hints, nops
+
+  # SMEM: memory loads (not ALU)
+  if inst_type is SMEM:
+    addr = st.rsgpr64(inst.sbase * 2) + _sext(inst.offset, 21)
+    if inst.soffset not in (NULL, 0x7f): addr += st.rsrc(inst.soffset, 0)
+    if (cnt := SMEM_LOAD.get(inst.op)) is None: raise NotImplementedError(f"SMEM op {inst.op}")
+    for i in range(cnt): st.wsgpr(inst.sdata + i, mem_read((addr + i * 4) & 0xffffffffffffffff, 4))
+    return 0
+
+  # SOP1: special handling for ops not in pseudocode
+  if inst_type is SOP1:
+    op = SOP1Op(inst.op)
+    # S_GETPC_B64: Get program counter (PC is stored as byte offset, convert from words)
+    if op == SOP1Op.S_GETPC_B64:
+      pc_bytes = st.pc * 4  # PC is in words, convert to bytes
+      st.wsgpr64(inst.sdst, pc_bytes)
+      return 0
+    # S_SETPC_B64: Set program counter to source value (indirect jump)
+    # Returns delta such that st.pc + inst_words + delta = target_words
+    if op == SOP1Op.S_SETPC_B64:
+      target_bytes = st.rsrc64(inst.ssrc0, 0)
+      target_words = target_bytes // 4
+      inst_words = 1  # SOP1 is always 1 word
+      return target_words - st.pc - inst_words
+
+  # Get op enum and lookup compiled function
+  if inst_type is SOP1: op_cls, ssrc0, sdst = SOP1Op, inst.ssrc0, inst.sdst
+  elif inst_type is SOP2: op_cls, ssrc0, sdst = SOP2Op, inst.ssrc0, inst.sdst
+  elif inst_type is SOPC: op_cls, ssrc0, sdst = SOPCOp, inst.ssrc0, None
+  elif inst_type is SOPK: op_cls, ssrc0, sdst = SOPKOp, inst.sdst, inst.sdst  # sdst is both src and dst
+  else: raise NotImplementedError(f"Unknown scalar type {inst_type}")
+
+  op = op_cls(inst.op)
+  fn = compiled.get(op_cls, {}).get(op)
+  if fn is None: raise NotImplementedError(f"{op.name} not in pseudocode")
+
+  # Build context - handle 64-bit ops that need 64-bit source reads
+  # 64-bit source ops: name ends with _B64, _I64, _U64 or contains _U64, _I64 before last underscore
+  is_64bit_s0 = op.name.endswith(('_B64', '_I64', '_U64')) or '_U64_' in op.name or '_I64_' in op.name
+  is_64bit_s0s1 = op_cls is SOPCOp and op in (SOPCOp.S_CMP_EQ_U64, SOPCOp.S_CMP_LG_U64)
+  s0 = st.rsrc64(ssrc0, 0) if is_64bit_s0 or is_64bit_s0s1 else (st.rsrc(ssrc0, 0) if inst_type != SOPK else st.rsgpr(inst.sdst))
+  is_64bit_sop2 = is_64bit_s0 and inst_type is SOP2
+  s1 = st.rsrc64(inst.ssrc1, 0) if (is_64bit_sop2 or is_64bit_s0s1) else (st.rsrc(inst.ssrc1, 0) if inst_type in (SOP2, SOPC) else inst.simm16 if inst_type is SOPK else 0)
+  d0 = st.rsgpr64(sdst) if (is_64bit_s0 or is_64bit_s0s1) and sdst is not None else (st.rsgpr(sdst) if sdst is not None else 0)
+  literal = inst.simm16 if inst_type is SOPK else st.literal
+
+  # Create Reg objects for new calling convention
+  S0, S1, S2, D0 = Reg(s0), Reg(s1), Reg(0), Reg(d0)
+  SCC, VCC, EXEC = Reg(st.scc), Reg(st.vcc), Reg(st.exec_mask)
+
+  # Execute compiled function - fn(S0, S1, S2, D0, SCC, VCC, laneId, EXEC, SIMM16, VGPR, SRC0, VDST)
+  fn(S0, S1, S2, D0, SCC, VCC, 0, EXEC, Reg(literal), None, 0, 0)
+
+  # Apply results from Reg objects
+  is_64bit_d0 = is_64bit_s0 or is_64bit_s0s1
+  if sdst is not None:
+    if is_64bit_d0:
+      st.wsgpr64(sdst, D0._val)
+    else:
+      st.wsgpr(sdst, D0._val)
+  st.scc = SCC._val
+  st.exec_mask = EXEC._val
+  return 0
+
+def exec_vector(st: WaveState, inst: Inst, lane: int, lds: bytearray | None = None,
+                d0_override: 'Reg | None' = None, vcc_override: 'Reg | None' = None) -> None:
+  """Execute vector instruction for one lane.
+  d0_override: For VOPC/VOP3-VOPC, use this Reg instead of st.sgpr[vdst] for D0 output.
+  vcc_override: For VOP3SD, use this Reg instead of st.sgpr[sdst] for VCC output.
+  """
+  compiled = _get_compiled()
+  inst_type, V = type(inst), st.vgpr[lane]
+
+  # Memory ops (not ALU pseudocode)
+  if inst_type is FLAT:
+    op, addr_reg, data_reg, vdst, offset, saddr = inst.op, inst.addr, inst.data, inst.vdst, _sext(inst.offset, 13), inst.saddr
+    addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+    addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+    if op in FLAT_LOAD:
+      cnt, sz, sign = FLAT_LOAD[op]
+      for i in range(cnt): val = mem_read(addr + i * sz, sz); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in FLAT_STORE:
+      cnt, sz = FLAT_STORE[op]
+      for i in range(cnt): mem_write(addr + i * sz, sz, V[data_reg + i]._val & ((1 << (sz * 8)) - 1))
+    elif op in FLAT_D16_LOAD:
+      sz, sign, hi = FLAT_D16_LOAD[op]
+      val = mem_read(addr, sz)
+      if sign: val = _sext(val, sz * 8) & 0xffff
+      if hi: V[vdst]._val = (V[vdst]._val & 0xffff) | (val << 16)
+      else: V[vdst]._val = (V[vdst]._val & 0xffff0000) | (val & 0xffff)
+    elif op in FLAT_D16_STORE:
+      sz, hi = FLAT_D16_STORE[op]
+      val = (V[data_reg]._val >> 16) & 0xffff if hi else V[data_reg]._val & 0xffff
+      mem_write(addr, sz, val & ((1 << (sz * 8)) - 1))
+    else: raise NotImplementedError(f"FLAT op {op}")
+    return
+
+  if inst_type is DS:
+    op, addr, vdst = inst.op, (V[inst.addr]._val + inst.offset0) & 0xffff, inst.vdst
+    if op in DS_LOAD:
+      cnt, sz, sign = DS_LOAD[op]
+      for i in range(cnt): val = int.from_bytes(lds[addr+i*sz:addr+i*sz+sz], 'little'); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in DS_STORE:
+      cnt, sz = DS_STORE[op]
+      for i in range(cnt): lds[addr+i*sz:addr+i*sz+sz] = (V[inst.data0 + i]._val & ((1 << (sz * 8)) - 1)).to_bytes(sz, 'little')
+    else: raise NotImplementedError(f"DS op {op}")
+    return
+
+  # VOPD: dual-issue, execute two ops using VOP2/VOP3 compiled functions
+  if inst_type is VOPD:
+    vdsty = (inst.vdsty << 1) | ((inst.vdstx & 1) ^ 1)
+    # Read all source operands BEFORE any writes (dual-issue semantics)
+    sx0, sx1 = Reg(st.rsrc(inst.srcx0, lane)), Reg(V[inst.vsrcx1]._val)
+    sy0, sy1 = Reg(st.rsrc(inst.srcy0, lane)), Reg(V[inst.vsrcy1]._val)
+    dx0, dy0 = Reg(V[inst.vdstx]._val), Reg(V[vdsty]._val)
+    st._scc_reg._val = st.scc
+    if (op_x := _VOPD_TO_VOP.get(inst.opx)):
+      if (fn_x := compiled.get(type(op_x), {}).get(op_x)):
+        fn_x(sx0, sx1, Reg(0), dx0, st._scc_reg, st.sgpr[VCC_LO], lane, st.sgpr[EXEC_LO], Reg(st.literal), None, Reg(0), Reg(inst.vdstx))
+    if (op_y := _VOPD_TO_VOP.get(inst.opy)):
+      if (fn_y := compiled.get(type(op_y), {}).get(op_y)):
+        fn_y(sy0, sy1, Reg(0), dy0, st._scc_reg, st.sgpr[VCC_LO], lane, st.sgpr[EXEC_LO], Reg(st.literal), None, Reg(0), Reg(vdsty))
+    V[inst.vdstx]._val, V[vdsty]._val = dx0._val, dy0._val
+    st.scc = st._scc_reg._val
+    return
+
+  # Determine instruction format and get function
+  is_vop3_vopc = False
+  is_readlane = False
+  if inst_type is VOP1:
+    if inst.op == VOP1Op.V_NOP: return
+    op_cls, op, src0, src1, src2, vdst = VOP1Op, VOP1Op(inst.op), inst.src0, None, None, inst.vdst
+    # V_READFIRSTLANE_B32 writes to SGPR, not VGPR
+    is_readlane = inst.op == VOP1Op.V_READFIRSTLANE_B32
+  elif inst_type is VOP2:
+    op_cls, op, src0, src1, src2, vdst = VOP2Op, VOP2Op(inst.op), inst.src0, inst.vsrc1 + 256, None, inst.vdst
+  elif inst_type is VOP3:
+    if inst.op < 256:
+      # VOP3-encoded VOPC - destination is an SGPR (vdst field)
+      op_cls, op, src0, src1, src2, vdst = VOPCOp, VOPCOp(inst.op), inst.src0, inst.src1, None, inst.vdst
+      is_vop3_vopc = True
+    else:
+      op_cls, op, src0, src1, src2, vdst = VOP3Op, VOP3Op(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+      # V_READFIRSTLANE_B32 and V_READLANE_B32 write to SGPR
+      is_readlane = inst.op in (VOP3Op.V_READFIRSTLANE_B32, VOP3Op.V_READLANE_B32)
+  elif inst_type is VOP3SD:
+    op_cls, op, src0, src1, src2, vdst = VOP3SDOp, VOP3SDOp(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+  elif inst_type is VOPC:
+    op_cls, op, src0, src1, src2, vdst = VOPCOp, VOPCOp(inst.op), inst.src0, inst.vsrc1 + 256, None, VCC_LO
+  elif inst_type is VOP3P:
+    op_cls, op, src0, src1, src2, vdst = VOP3POp, VOP3POp(inst.op), inst.src0, inst.src1, inst.src2, inst.vdst
+    # WMMA instructions are handled specially (only execute for lane 0)
+    if op in (VOP3POp.V_WMMA_F32_16X16X16_F16, VOP3POp.V_WMMA_F16_16X16X16_F16):
+      if lane == 0: exec_wmma(st, inst, op)
+      return
+  else: raise NotImplementedError(f"Unknown vector type {inst_type}")
+
+  fn = compiled.get(op_cls, {}).get(op)
+  if fn is None: raise NotImplementedError(f"{op.name} not in pseudocode")
+
+  # Build source Regs - get the actual register or create temp for inline constants
+  # VOP3P uses f16 inline constants (16-bit value in low half only)
+  if inst_type is VOP3P:
+    S0 = st.rsrc_reg_f16(src0, lane)
+    S1 = st.rsrc_reg_f16(src1, lane) if src1 is not None else Reg(0)
+    S2 = st.rsrc_reg_f16(src2, lane) if src2 is not None else Reg(0)
+    # Apply op_sel_hi modifiers: control which half is used for hi-half computation
+    # opsel_hi[0]=0 means src0 hi comes from lo half, =1 means from hi half (default)
+    # opsel_hi[1]=0 means src1 hi comes from lo half, =1 means from hi half (default)
+    # opsel_hi2=0 means src2 hi comes from lo half, =1 means from hi half (default)
+    opsel_hi = getattr(inst, 'opsel_hi', 3)  # default 0b11
+    opsel_hi2 = getattr(inst, 'opsel_hi2', 1)  # default 1
+    # If opsel_hi bit is 0, replicate lo half to hi half
+    if not (opsel_hi & 1):  # src0 hi from lo
+      lo = S0._val & 0xffff
+      S0 = Reg((lo << 16) | lo)
+    if not (opsel_hi & 2):  # src1 hi from lo
+      lo = S1._val & 0xffff
+      S1 = Reg((lo << 16) | lo)
+    if not opsel_hi2:  # src2 hi from lo
+      lo = S2._val & 0xffff
+      S2 = Reg((lo << 16) | lo)
+  else:
+    # Check if this is a 64-bit F64 op - needs 64-bit source reads for f64 operands
+    # V_LDEXP_F64: S0 is f64, S1 is i32 (exponent)
+    # V_ADD_F64, V_MUL_F64, etc: S0 and S1 are f64
+    # VOP1 F64 ops (V_TRUNC_F64, V_FLOOR_F64, etc): S0 is f64
+    is_f64_op = hasattr(op, 'name') and '_F64' in op.name
+    is_ldexp_f64 = hasattr(op, 'name') and op.name == 'V_LDEXP_F64'
+    if is_f64_op:
+      S0 = st.rsrc_reg64(src0, lane)
+      # V_LDEXP_F64: S1 is i32 exponent, not f64
+      if is_ldexp_f64:
+        S1 = st.rsrc_reg(src1, lane) if src1 is not None else Reg(0)
+      else:
+        S1 = st.rsrc_reg64(src1, lane) if src1 is not None else Reg(0)
+      S2 = st.rsrc_reg64(src2, lane) if src2 is not None else Reg(0)
+    else:
+      S0 = st.rsrc_reg(src0, lane)
+      S1 = st.rsrc_reg(src1, lane) if src1 is not None else Reg(0)
+      S2 = st.rsrc_reg(src2, lane) if src2 is not None else Reg(0)
+    # VOP3SD V_MAD_U64_U32 and V_MAD_I64_I32 need S2 as 64-bit from VGPR pair
+    if inst_type is VOP3SD and op in (VOP3SDOp.V_MAD_U64_U32, VOP3SDOp.V_MAD_I64_I32) and src2 is not None:
+      if 256 <= src2 <= 511:  # VGPR
+        vgpr_idx = src2 - 256
+        S2 = Reg(V[vgpr_idx]._val | (V[vgpr_idx + 1]._val << 32))
+
+  # Apply source modifiers (neg, abs) for VOP3/VOP3SD
+  if inst_type in (VOP3, VOP3SD):
+    neg, abs_mod = getattr(inst, 'neg', 0), getattr(inst, 'abs', 0)
+    if neg or abs_mod:
+      # Apply to f32 values - need to handle as float
+      import struct
+      def apply_mods(reg, neg_bit, abs_bit):
+        val = reg._val
+        f = struct.unpack('<f', struct.pack('<I', val & 0xffffffff))[0]
+        if abs_bit: f = abs(f)
+        if neg_bit: f = -f
+        return Reg(struct.unpack('<I', struct.pack('<f', f))[0])
+      if neg & 1 or abs_mod & 1: S0 = apply_mods(S0, neg & 1, abs_mod & 1)
+      if neg & 2 or abs_mod & 2: S1 = apply_mods(S1, neg & 2, abs_mod & 2)
+      if neg & 4 or abs_mod & 4: S2 = apply_mods(S2, neg & 4, abs_mod & 4)
+
+  # Apply opsel for VOP3 f16 operations - select which half to use
+  # opsel[0]: src0, opsel[1]: src1, opsel[2]: src2 (0=lo, 1=hi)
+  if inst_type is VOP3:
+    opsel = getattr(inst, 'opsel', 0)
+    if opsel:
+      # If opsel bit is set, swap lo and hi so that .f16 reads the hi half
+      if opsel & 1:  # src0 from hi
+        S0 = Reg(((S0._val >> 16) & 0xffff) | (S0._val << 16))
+      if opsel & 2:  # src1 from hi
+        S1 = Reg(((S1._val >> 16) & 0xffff) | (S1._val << 16))
+      if opsel & 4:  # src2 from hi
+        S2 = Reg(((S2._val >> 16) & 0xffff) | (S2._val << 16))
+
+  # For VOPC and VOP3-encoded VOPC, D0 is an SGPR (VCC_LO for VOPC, vdst for VOP3 VOPC)
+  # V_READFIRSTLANE_B32 and V_READLANE_B32 also write to SGPR
+  # Use d0_override if provided (for batch execution with shared output register)
+  is_vopc = inst_type is VOPC or (inst_type is VOP3 and is_vop3_vopc)
+  if is_vopc:
+    D0 = d0_override if d0_override is not None else st.sgpr[VCC_LO if inst_type is VOPC else vdst]
+  elif is_readlane:
+    D0 = st.sgpr[vdst]
+  else:
+    D0 = V[vdst]
+
+  # Execute compiled function - D0 is modified in place
+  st._scc_reg._val = st.scc
+  # For VOP3SD, pass sdst register as VCC parameter (carry-out destination)
+  # Use vcc_override if provided (for batch execution with shared output register)
+  # For VOP3 V_CNDMASK_B32, src2 specifies the condition selector (not VCC)
+  if inst_type is VOP3SD:
+    vcc_reg = vcc_override if vcc_override is not None else st.sgpr[inst.sdst]
+  elif inst_type is VOP3 and op == VOP3Op.V_CNDMASK_B32 and src2 is not None:
+    vcc_reg = st.rsrc_reg(src2, lane)  # Use src2 as condition
+  else:
+    vcc_reg = st.sgpr[VCC_LO]
+  # SRC0/VDST are VGPR indices (0-255), not hardware encoding (256-511)
+  src0_idx = (src0 - 256) if src0 and src0 >= 256 else (src0 if src0 else 0)
+  result = fn(S0, S1, S2, D0, st._scc_reg, vcc_reg, lane, st.sgpr[EXEC_LO], Reg(st.literal), st.vgpr, Reg(src0_idx), Reg(vdst))
+  st.scc = st._scc_reg._val
+
+  # Handle special results
+  if result:
+    if 'vgpr_write' in result:
+      wr_lane, wr_idx, wr_val = result['vgpr_write']
+      st.vgpr[wr_lane][wr_idx]._val = wr_val
+
+  # 64-bit destination: write high 32 bits to next VGPR (determined from op name)
+  is_64bit_dst = not is_vopc and not is_readlane and hasattr(op, 'name') and \
+                 any(s in op.name for s in ('_B64', '_I64', '_U64', '_F64'))
+  if is_64bit_dst:
+    V[vdst + 1]._val = (D0._val >> 32) & 0xffffffff
+    D0._val = D0._val & 0xffffffff  # Keep only low 32 bits in D0
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# WMMA (Wave Matrix Multiply-Accumulate)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def exec_wmma(st: WaveState, inst, op: VOP3POp) -> None:
+  """Execute WMMA instruction - 16x16x16 matrix multiply across the wave."""
+  src0, src1, src2, vdst = inst.src0, inst.src1, inst.src2, inst.vdst
+  # Read matrix A (16x16 f16/bf16) from lanes 0-15, VGPRs src0 to src0+7 (2 f16 per VGPR = 16 values per lane)
+  # Layout: A[row][k] where row = lane (0-15), k comes from 8 VGPRs × 2 halves
+  mat_a = []
+  for lane in range(16):
+    for reg in range(8):
+      val = st.vgpr[lane][src0 - 256 + reg] if src0 >= 256 else st.rsgpr(src0 + reg)
+      mat_a.append(_f16(val & 0xffff))
+      mat_a.append(_f16((val >> 16) & 0xffff))
+  # Read matrix B (16x16 f16/bf16) - same layout, B[col][k] where col comes from lane
+  mat_b = []
+  for lane in range(16):
+    for reg in range(8):
+      val = st.vgpr[lane][src1 - 256 + reg] if src1 >= 256 else st.rsgpr(src1 + reg)
+      mat_b.append(_f16(val & 0xffff))
+      mat_b.append(_f16((val >> 16) & 0xffff))
+
+  # Read matrix C (16x16 f32) from lanes 0-31, VGPRs src2 to src2+7
+  # Layout: element i is at lane (i % 32), VGPR (i // 32) + src2
+  mat_c = []
+  for i in range(256):
+    lane, reg = i % 32, i // 32
+    val = st.vgpr[lane][src2 - 256 + reg] if src2 >= 256 else st.rsgpr(src2 + reg)
+    mat_c.append(_f32(val))
+
+  # Compute D = A × B + C (16x16 matrix multiply)
+  mat_d = [0.0] * 256
+  for row in range(16):
+    for col in range(16):
+      acc = 0.0
+      for k in range(16):
+        a_val = mat_a[row * 16 + k]
+        b_val = mat_b[col * 16 + k]
+        acc += a_val * b_val
+      mat_d[row * 16 + col] = acc + mat_c[row * 16 + col]
+
+  # Write result matrix D back - same layout as C
+  if op == VOP3POp.V_WMMA_F16_16X16X16_F16:
+    # Output is f16, pack 2 values per VGPR
+    for i in range(0, 256, 2):
+      lane, reg = (i // 2) % 32, (i // 2) // 32
+      lo = _i16(mat_d[i]) & 0xffff
+      hi = _i16(mat_d[i + 1]) & 0xffff
+      st.vgpr[lane][vdst + reg]._val = (hi << 16) | lo
+  else:
+    # Output is f32
+    for i in range(256):
+      lane, reg = i % 32, i // 32
+      st.vgpr[lane][vdst + reg]._val = _i32(mat_d[i])
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# MAIN EXECUTION LOOP
+# ═══════════════════════════════════════════════════════════════════════════════
+
+SCALAR_TYPES = {SOP1, SOP2, SOPC, SOPK, SOPP, SMEM}
+VECTOR_TYPES = {VOP1, VOP2, VOP3, VOP3SD, VOPC, FLAT, DS, VOPD, VOP3P}
+
+# Pre-cache compiled functions for fast lookup
+_COMPILED_CACHE: dict | None = None
+def _get_fn(op_cls, op):
+  global _COMPILED_CACHE
+  if _COMPILED_CACHE is None: _COMPILED_CACHE = _get_compiled()
+  return _COMPILED_CACHE.get(op_cls, {}).get(op)
+
+def exec_vector_batch(st: WaveState, inst: Inst, exec_mask: int, n_lanes: int, lds: bytearray | None = None) -> None:
+  """Execute vector instruction for all active lanes at once."""
+  compiled = _get_compiled()
+  inst_type = type(inst)
+  vgpr = st.vgpr
+
+  # Memory ops - still per-lane but inlined
+  if inst_type is FLAT:
+    op, addr_reg, data_reg, vdst, offset, saddr = inst.op, inst.addr, inst.data, inst.vdst, _sext(inst.offset, 13), inst.saddr
+    if op in FLAT_LOAD:
+      cnt, sz, sign = FLAT_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        for i in range(cnt): val = mem_read(addr + i * sz, sz); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in FLAT_STORE:
+      cnt, sz = FLAT_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        for i in range(cnt): mem_write(addr + i * sz, sz, V[data_reg + i]._val & ((1 << (sz * 8)) - 1))
+    elif op in FLAT_D16_LOAD:
+      sz, sign, hi = FLAT_D16_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        val = mem_read(addr, sz)
+        if sign: val = _sext(val, sz * 8) & 0xffff
+        if hi: V[vdst]._val = (V[vdst]._val & 0xffff) | (val << 16)
+        else: V[vdst]._val = (V[vdst]._val & 0xffff0000) | (val & 0xffff)
+    elif op in FLAT_D16_STORE:
+      sz, hi = FLAT_D16_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = V[addr_reg]._val | (V[addr_reg+1]._val << 32)
+        addr = (st.rsgpr64(saddr) + V[addr_reg]._val + offset) & 0xffffffffffffffff if saddr not in (NULL, 0x7f) else (addr + offset) & 0xffffffffffffffff
+        val = (V[data_reg]._val >> 16) & 0xffff if hi else V[data_reg]._val & 0xffff
+        mem_write(addr, sz, val & ((1 << (sz * 8)) - 1))
+    else: raise NotImplementedError(f"FLAT op {op}")
+    return
+
+  if inst_type is DS:
+    op, vdst = inst.op, inst.vdst
+    if op in DS_LOAD:
+      cnt, sz, sign = DS_LOAD[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = (V[inst.addr]._val + inst.offset0) & 0xffff
+        for i in range(cnt): val = int.from_bytes(lds[addr+i*sz:addr+i*sz+sz], 'little'); V[vdst + i]._val = _sext(val, sz * 8) & 0xffffffff if sign else val
+    elif op in DS_STORE:
+      cnt, sz = DS_STORE[op]
+      for lane in range(n_lanes):
+        if not (exec_mask & (1 << lane)): continue
+        V = vgpr[lane]
+        addr = (V[inst.addr]._val + inst.offset0) & 0xffff
+        for i in range(cnt): lds[addr+i*sz:addr+i*sz+sz] = (V[inst.data0 + i]._val & ((1 << (sz * 8)) - 1)).to_bytes(sz, 'little')
+    else: raise NotImplementedError(f"DS op {op}")
+    return
+
+  # For VOPC, VOP3-encoded VOPC, and VOP3SD, we write per-lane bits to an SGPR.
+  # The pseudocode does D0.u64[laneId] = bit or VCC.u64[laneId] = bit.
+  # To avoid corrupting reads from the same SGPR, use a shared output Reg(0).
+  # Exception: CMPX instructions write to EXEC (not D0/VCC).
+  d0_override, vcc_override = None, None
+  vopc_dst, vop3sd_dst = None, None
+  is_cmpx = False
+  if inst_type is VOPC:
+    op = VOPCOp(inst.op)
+    is_cmpx = 'CMPX' in op.name
+    if not is_cmpx:  # Regular CMP writes to VCC
+      d0_override, vopc_dst = Reg(0), VCC_LO
+    else:  # CMPX writes to EXEC - clear it first, accumulate per-lane
+      st.sgpr[EXEC_LO]._val = 0
+  elif inst_type is VOP3 and inst.op < 256:  # VOP3-encoded VOPC
+    op = VOPCOp(inst.op)
+    is_cmpx = 'CMPX' in op.name
+    if not is_cmpx:  # Regular CMP writes to destination SGPR
+      d0_override, vopc_dst = Reg(0), inst.vdst
+    else:  # CMPX writes to EXEC - clear it first, accumulate per-lane
+      st.sgpr[EXEC_LO]._val = 0
+  if inst_type is VOP3SD:
+    vcc_override, vop3sd_dst = Reg(0), inst.sdst
+
+  # For other vector ops, dispatch to exec_vector per lane (can optimize later)
+  for lane in range(n_lanes):
+    if exec_mask & (1 << lane): exec_vector(st, inst, lane, lds, d0_override, vcc_override)
+
+  # Write accumulated per-lane bit results to destination SGPRs
+  # (CMPX writes directly to EXEC in the pseudocode, so no separate write needed)
+  if vopc_dst is not None: st.sgpr[vopc_dst]._val = d0_override._val
+  if vop3sd_dst is not None: st.sgpr[vop3sd_dst]._val = vcc_override._val
+
+def step_wave(program: Program, st: WaveState, lds: bytearray, n_lanes: int) -> int:
+  inst = program.get(st.pc)
+  if inst is None: return 1
+  inst_words, st.literal, inst_type = inst._words, getattr(inst, '_literal', None) or 0, type(inst)
+
+  if inst_type in SCALAR_TYPES:
+    delta = exec_scalar(st, inst)
+    if delta == -1: return -1  # endpgm
+    if delta == -2: st.pc += inst_words; return -2  # barrier
+    st.pc += inst_words + delta
+  else:
+    # V_READFIRSTLANE_B32 and V_READLANE_B32 write to SGPR, so they should only execute once per wave (lane 0)
+    is_readlane = (inst_type is VOP1 and inst.op == VOP1Op.V_READFIRSTLANE_B32) or \
+                  (inst_type is VOP3 and inst.op in (VOP3Op.V_READFIRSTLANE_B32, VOP3Op.V_READLANE_B32))
+    if is_readlane:
+      exec_vector(st, inst, 0, lds)  # Execute once with lane 0
+    else:
+      exec_vector_batch(st, inst, st.exec_mask, n_lanes, lds)
+    st.commit_pends()
+    st.pc += inst_words
+  return 0
+
+def exec_wave(program: Program, st: WaveState, lds: bytearray, n_lanes: int) -> int:
+  while st.pc in program:
+    result = step_wave(program, st, lds, n_lanes)
+    if result == -1: return 0
+    if result == -2: return -2
+  return 0
+
+def exec_workgroup(program: Program, workgroup_id: tuple[int, int, int], local_size: tuple[int, int, int], args_ptr: int,
+                   wg_id_sgpr_base: int, wg_id_enables: tuple[bool, bool, bool]) -> None:
+  lx, ly, lz = local_size
+  total_threads, lds = lx * ly * lz, bytearray(65536)
+  waves: list[tuple[WaveState, int, int]] = []
+  for wave_start in range(0, total_threads, WAVE_SIZE):
+    n_lanes, st = min(WAVE_SIZE, total_threads - wave_start), WaveState()
+    st.exec_mask = (1 << n_lanes) - 1
+    st.wsgpr64(0, args_ptr)
+    gx, gy, gz = workgroup_id
+    # Set workgroup IDs in SGPRs based on USER_SGPR_COUNT and enable flags from COMPUTE_PGM_RSRC2
+    sgpr_idx = wg_id_sgpr_base
+    if wg_id_enables[0]: st.sgpr[sgpr_idx]._val = gx; sgpr_idx += 1
+    if wg_id_enables[1]: st.sgpr[sgpr_idx]._val = gy; sgpr_idx += 1
+    if wg_id_enables[2]: st.sgpr[sgpr_idx]._val = gz
+    for i in range(n_lanes):
+      tid = wave_start + i
+      st.vgpr[i][0]._val = tid if local_size == (lx, 1, 1) else ((tid // (lx * ly)) << 20) | (((tid // lx) % ly) << 10) | (tid % lx)
+    waves.append((st, n_lanes, wave_start))
+  has_barrier = any(isinstance(inst, SOPP) and inst.op == SOPPOp.S_BARRIER for inst in program.values())
+  for _ in range(2 if has_barrier else 1):
+    for st, n_lanes, _ in waves: exec_wave(program, st, lds, n_lanes)
+
+def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c) -> int:
+  data = (ctypes.c_char * lib_sz).from_address(lib).raw
+  program = decode_program(data)
+  if not program: return -1
+  # Parse COMPUTE_PGM_RSRC2 for SGPR layout
+  user_sgpr_count = (rsrc2 >> 1) & 0x1f
+  enable_wg_id_x = bool((rsrc2 >> 7) & 1)
+  enable_wg_id_y = bool((rsrc2 >> 8) & 1)
+  enable_wg_id_z = bool((rsrc2 >> 9) & 1)
+  wg_id_enables = (enable_wg_id_x, enable_wg_id_y, enable_wg_id_z)
+  for gidz in range(gz):
+    for gidy in range(gy):
+      for gidx in range(gx): exec_workgroup(program, (gidx, gidy, gidz), (lx, ly, lz), args_ptr, user_sgpr_count, wg_id_enables)
+  return 0

extra/assembly/amd/test/bench_emu.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""Benchmark comparing Python vs Rust RDNA3 emulators on synthetic and real tinygrad kernels."""
+import ctypes, time, os, struct, cProfile, pstats, io
+from pathlib import Path
+from typing import Callable
+
+# Set AMD=1 before importing tinygrad
+os.environ["AMD"] = "1"
+
+from extra.assembly.amd.emu import run_asm as python_run_asm, set_valid_mem_ranges, decode_program, step_wave, WaveState, WAVE_SIZE
+
+REMU_PATH = Path(__file__).parents[3] / "remu/target/release/libremu.so"
+if not REMU_PATH.exists():
+  REMU_PATH = Path(__file__).parents[3] / "remu/target/release/libremu.dylib"
+
+def get_rust_remu():
+  """Load the Rust libremu shared library."""
+  if not REMU_PATH.exists(): return None
+  remu = ctypes.CDLL(str(REMU_PATH))
+  remu.run_asm.restype = ctypes.c_int32
+  remu.run_asm.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32,
+                           ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_void_p]
+  return remu
+
+def count_instructions(kernel: bytes) -> int:
+  """Count instructions in a kernel."""
+  return len(decode_program(kernel))
+
+def setup_buffers(buf_sizes: list[int], init_data: dict[int, bytes] | None = None):
+  """Allocate buffers and return args pointer + valid ranges."""
+  if init_data is None: init_data = {}
+  buffers = []
+  for i, size in enumerate(buf_sizes):
+    padded = ((size + 15) // 16) * 16 + 16
+    data = init_data.get(i, b'\x00' * padded)
+    data_list = list(data) + [0] * (padded - len(data))
+    buf = (ctypes.c_uint8 * padded)(*data_list[:padded])
+    buffers.append(buf)
+  args = (ctypes.c_uint64 * len(buffers))(*[ctypes.addressof(b) for b in buffers])
+  args_ptr = ctypes.addressof(args)
+  ranges = {(ctypes.addressof(b), len(b)) for b in buffers}
+  ranges.add((args_ptr, ctypes.sizeof(args)))
+  return buffers, args, args_ptr, ranges
+
+def benchmark_emulator(name: str, run_fn, kernel: bytes, global_size, local_size, args_ptr, iterations: int = 5):
+  """Benchmark an emulator and return average time."""
+  gx, gy, gz = global_size
+  lx, ly, lz = local_size
+  kernel_buf = (ctypes.c_char * len(kernel)).from_buffer_copy(kernel)
+  lib_ptr = ctypes.addressof(kernel_buf)
+
+  # Warmup
+  run_fn(lib_ptr, len(kernel), gx, gy, gz, lx, ly, lz, args_ptr)
+
+  # Timed runs
+  times = []
+  for _ in range(iterations):
+    start = time.perf_counter()
+    result = run_fn(lib_ptr, len(kernel), gx, gy, gz, lx, ly, lz, args_ptr)
+    end = time.perf_counter()
+    if result != 0:
+      print(f"  {name} returned error: {result}")
+      return None
+    times.append(end - start)
+
+  return sum(times) / len(times)
+
+def create_synthetic_kernel(n_ops: int) -> bytes:
+  """Create a synthetic kernel with n_ops vector operations."""
+  instructions = []
+  # VOP2 instructions: v_add_f32, v_mul_f32, v_max_f32, v_min_f32
+  ops = [
+    (0b0000011 << 25) | (1 << 17) | (0 << 9) | 256,  # v_add_f32 v0, v0, v1
+    (0b0001000 << 25) | (1 << 17) | (0 << 9) | 256,  # v_mul_f32 v0, v0, v1
+    (0b0010000 << 25) | (1 << 17) | (0 << 9) | 256,  # v_max_f32 v0, v0, v1
+    (0b0001111 << 25) | (1 << 17) | (0 << 9) | 256,  # v_min_f32 v0, v0, v1
+  ]
+  for i in range(n_ops):
+    instructions.append(ops[i % len(ops)])
+  # S_ENDPGM
+  instructions.append((0b101111111 << 23) | (48 << 16) | 0)
+  return b''.join(struct.pack('<I', inst) for inst in instructions)
+
+def get_tinygrad_kernel(op_name: str) -> tuple[bytes, tuple, tuple, list[int], dict[int, bytes]] | None:
+  """Get a real tinygrad kernel by operation name. Returns (code, global_size, local_size, buf_sizes, buf_data)."""
+  try:
+    from tinygrad import Tensor
+    from tinygrad.runtime.support.elf import elf_loader
+    import numpy as np
+    np.random.seed(42)
+
+    ops = {
+      "add": lambda: Tensor.empty(1024) + Tensor.empty(1024),
+      "mul": lambda: Tensor.empty(1024) * Tensor.empty(1024),
+      "matmul_small": lambda: Tensor.empty(16, 16) @ Tensor.empty(16, 16),
+      "matmul_medium": lambda: Tensor.empty(64, 64) @ Tensor.empty(64, 64),
+      "reduce_sum": lambda: Tensor.empty(4096).sum(),
+      "reduce_max": lambda: Tensor.empty(4096).max(),
+      "softmax": lambda: Tensor.empty(256).softmax(),
+      "layernorm": lambda: Tensor.empty(32, 64).layernorm(),
+      "conv2d": lambda: Tensor.empty(1, 4, 16, 16).conv2d(Tensor.empty(4, 4, 3, 3)),
+      "gelu": lambda: Tensor.empty(1024).gelu(),
+      "exp": lambda: Tensor.empty(1024).exp(),
+      "sin": lambda: Tensor.empty(1024).sin(),
+    }
+
+    if op_name not in ops: return None
+    out = ops[op_name]()
+    sched = out.schedule()
+
+    for ei in sched:
+      lowered = ei.lower()
+      if ei.ast.op.name == 'SINK' and lowered.prg and lowered.prg.p.lib:
+        lib = bytes(lowered.prg.p.lib)
+        _, sections, _ = elf_loader(lib)
+        for sec in sections:
+          if sec.name == '.text':
+            buf_sizes = [b.nbytes for b in lowered.bufs]
+            # Get initial data from numpy arrays if available
+            buf_data = {}
+            for i, buf in enumerate(lowered.bufs):
+              if hasattr(buf, 'base') and buf.base is not None and hasattr(buf.base, '_buf'):
+                try: buf_data[i] = bytes(buf.base._buf)
+                except: pass
+            return (bytes(sec.content), tuple(lowered.prg.p.global_size), tuple(lowered.prg.p.local_size), buf_sizes, buf_data)
+    return None
+  except Exception as e:
+    print(f"  Error getting kernel: {e}")
+    return None
+
+def profile_python_emu(kernel: bytes, global_size, local_size, args_ptr, n_runs: int = 1):
+  """Profile the Python emulator to find bottlenecks."""
+  gx, gy, gz = global_size
+  lx, ly, lz = local_size
+  kernel_buf = (ctypes.c_char * len(kernel)).from_buffer_copy(kernel)
+  lib_ptr = ctypes.addressof(kernel_buf)
+
+  pr = cProfile.Profile()
+  pr.enable()
+  for _ in range(n_runs):
+    python_run_asm(lib_ptr, len(kernel), gx, gy, gz, lx, ly, lz, args_ptr)
+  pr.disable()
+
+  s = io.StringIO()
+  ps = pstats.Stats(pr, stream=s).sort_stats('cumulative')
+  ps.print_stats(20)
+  return s.getvalue()
+
+def measure_step_rate(kernel: bytes, n_steps: int = 10000) -> float:
+  """Measure raw step_wave() performance (steps per second)."""
+  program = decode_program(kernel)
+  if not program: return 0.0
+
+  st = WaveState()
+  st.exec_mask = 0xffffffff
+  lds = bytearray(65536)
+  n_lanes = 32
+
+  # Reset PC for each measurement
+  start = time.perf_counter()
+  for _ in range(n_steps):
+    st.pc = 0
+    while st.pc in program:
+      result = step_wave(program, st, lds, n_lanes)
+      if result == -1: break
+  elapsed = time.perf_counter() - start
+  return n_steps / elapsed if elapsed > 0 else 0
+
+# Test configurations
+SYNTHETIC_TESTS = [
+  ("synthetic_10ops", 10, (1, 1, 1), (32, 1, 1)),
+  ("synthetic_100ops", 100, (1, 1, 1), (32, 1, 1)),
+  ("synthetic_500ops", 500, (1, 1, 1), (32, 1, 1)),
+  ("synthetic_100ops_4wg", 100, (4, 1, 1), (32, 1, 1)),
+  ("synthetic_100ops_16wg", 100, (16, 1, 1), (32, 1, 1)),
+]
+
+TINYGRAD_TESTS = ["add", "mul", "reduce_sum", "softmax", "exp", "gelu", "matmul_small"]
+
+def main():
+  import argparse
+  parser = argparse.ArgumentParser(description="Benchmark RDNA3 emulators")
+  parser.add_argument("--profile", action="store_true", help="Profile Python emulator")
+  parser.add_argument("--synthetic-only", action="store_true", help="Only run synthetic tests")
+  parser.add_argument("--tinygrad-only", action="store_true", help="Only run tinygrad tests")
+  parser.add_argument("--iterations", type=int, default=3, help="Number of iterations per benchmark")
+  args = parser.parse_args()
+
+  rust_remu = get_rust_remu()
+  if rust_remu is None:
+    print("Rust libremu not found. Build with: cargo build --release --manifest-path extra/remu/Cargo.toml")
+    print("Running Python-only benchmarks...\n")
+
+  print("=" * 90)
+  print("RDNA3 Emulator Benchmark: Python vs Rust")
+  print("=" * 90)
+
+  results = []
+
+  # Synthetic workloads
+  if not args.tinygrad_only:
+    print("\n[SYNTHETIC WORKLOADS]")
+    print("-" * 90)
+
+    for name, n_ops, global_size, local_size in SYNTHETIC_TESTS:
+      kernel = create_synthetic_kernel(n_ops)
+      n_insts = count_instructions(kernel)
+      n_workgroups = global_size[0] * global_size[1] * global_size[2]
+      n_threads = local_size[0] * local_size[1] * local_size[2]
+      total_work = n_insts * n_workgroups * n_threads
+
+      print(f"\n{name}: {n_insts} insts × {n_workgroups} WGs × {n_threads} threads = {total_work:,} ops")
+
+      buf_sizes = [4096]
+      buffers, args_arr, args_ptr, ranges = setup_buffers(buf_sizes)
+      set_valid_mem_ranges(ranges)
+
+      # Benchmark
+      py_time = benchmark_emulator("Python", python_run_asm, kernel, global_size, local_size, args_ptr, args.iterations)
+      rust_time = benchmark_emulator("Rust", rust_remu.run_asm, kernel, global_size, local_size, args_ptr, args.iterations) if rust_remu else None
+
+      if py_time:
+        py_rate = total_work / py_time / 1e6
+        print(f"  Python: {py_time*1000:8.3f} ms  ({py_rate:7.2f} M ops/s)")
+      if rust_time:
+        rust_rate = total_work / rust_time / 1e6
+        speedup = py_time / rust_time if py_time else 0
+        print(f"  Rust:   {rust_time*1000:8.3f} ms  ({rust_rate:7.2f} M ops/s)  [{speedup:.1f}x faster]")
+
+      results.append(("synthetic", name, n_insts, n_workgroups, py_time, rust_time))
+
+  # Tinygrad kernels
+  if not args.synthetic_only:
+    print("\n[TINYGRAD KERNELS]")
+    print("-" * 90)
+
+    for op_name in TINYGRAD_TESTS:
+      print(f"\n{op_name}:", end=" ", flush=True)
+      kernel_info = get_tinygrad_kernel(op_name)
+      if kernel_info is None:
+        print("failed to compile")
+        continue
+
+      kernel, global_size, local_size, buf_sizes, buf_data = kernel_info
+      n_insts = count_instructions(kernel)
+      n_workgroups = global_size[0] * global_size[1] * global_size[2]
+      n_threads = local_size[0] * local_size[1] * local_size[2]
+      total_work = n_insts * n_workgroups * n_threads
+
+      print(f"{n_insts} insts × {n_workgroups} WGs × {n_threads} threads = {total_work:,} ops")
+
+      buffers, args_arr, args_ptr, ranges = setup_buffers(buf_sizes, buf_data)
+      set_valid_mem_ranges(ranges)
+
+      py_time = benchmark_emulator("Python", python_run_asm, kernel, global_size, local_size, args_ptr, args.iterations)
+      rust_time = benchmark_emulator("Rust", rust_remu.run_asm, kernel, global_size, local_size, args_ptr, args.iterations) if rust_remu else None
+
+      if py_time:
+        py_rate = total_work / py_time / 1e6
+        print(f"  Python: {py_time*1000:8.3f} ms  ({py_rate:7.2f} M ops/s)")
+      if rust_time:
+        rust_rate = total_work / rust_time / 1e6
+        speedup = py_time / rust_time if py_time else 0
+        print(f"  Rust:   {rust_time*1000:8.3f} ms  ({rust_rate:7.2f} M ops/s)  [{speedup:.1f}x faster]")
+
+      results.append(("tinygrad", op_name, n_insts, n_workgroups, py_time, rust_time))
+
+      # Optional profiling
+      if args.profile and py_time:
+        print("\n  [PROFILE - Top 10 functions]")
+        profile_output = profile_python_emu(kernel, global_size, local_size, args_ptr)
+        for line in profile_output.split('\n')[5:15]:
+          if line.strip(): print(f"    {line}")
+
+  # Summary table
+  print("\n" + "=" * 90)
+  print("SUMMARY")
+  print("=" * 90)
+  print(f"{'Type':<10} {'Name':<25} {'Insts':<8} {'WGs':<6} {'Python (ms)':<14} {'Rust (ms)':<14} {'Speedup':<10}")
+  print("-" * 90)
+
+  for test_type, name, n_insts, n_wgs, py_time, rust_time in results:
+    py_ms = f"{py_time*1000:.3f}" if py_time else "error"
+    if rust_time:
+      rust_ms = f"{rust_time*1000:.3f}"
+      speedup = f"{py_time/rust_time:.1f}x" if py_time else "N/A"
+    else:
+      rust_ms, speedup = "N/A", "N/A"
+    print(f"{test_type:<10} {name:<25} {n_insts:<8} {n_wgs:<6} {py_ms:<14} {rust_ms:<14} {speedup:<10}")
+
+
+
+if __name__ == "__main__":
+  main()

extra/assembly/amd/test/external_test_usability.py

@@ -0,0 +1,196 @@
+# Usability tests for the RDNA3 ASM DSL
+# These tests demonstrate how the DSL *should* work for a good user experience
+# Currently many of these tests fail - they document desired behavior
+
+import unittest
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.dsl import Inst, RawImm, SGPR, VGPR
+
+class TestRegisterSliceSyntax(unittest.TestCase):
+  """
+  Issue: Register slice syntax should use AMD assembly convention (inclusive end).
+
+  In AMD assembly, s[4:7] means registers s4, s5, s6, s7 (4 registers, inclusive).
+  The DSL should match this convention so that:
+  - s[4:7] gives 4 registers
+  - Disassembler output can be copied directly back into DSL code
+
+  Fix: Change _RegFactory.__getitem__ to use inclusive end:
+    key.stop - key.start + 1  (instead of key.stop - key.start)
+  """
+  def test_register_slice_count(self):
+    # s[4:7] should give 4 registers: s4, s5, s6, s7 (AMD convention, inclusive)
+    reg = s[4:7]
+    self.assertEqual(reg.count, 4, "s[4:7] should give 4 registers (s4, s5, s6, s7)")
+
+  def test_register_slice_roundtrip(self):
+    # Round-trip: DSL -> disasm -> DSL should preserve register count
+    reg = s[4:7]  # 4 registers in AMD convention
+    inst = s_load_b128(reg, s[0:1], NULL, 0)
+    disasm = inst.disasm()
+    # Disasm shows s[4:7] - user should be able to copy this back
+    self.assertIn("s[4:7]", disasm)
+    # And s[4:7] in DSL should give the same 4 registers
+    reg_from_disasm = s[4:7]
+    self.assertEqual(reg_from_disasm.count, 4, "s[4:7] from disasm should give 4 registers")
+
+
+class TestReprReadability(unittest.TestCase):
+  """
+  Issue: repr() leaks internal RawImm type and omits zero-valued fields.
+
+  When you create v_mov_b32_e32(v[0], v[1]), the repr shows:
+    VOP1(op=1, src0=RawImm(257))
+
+  Problems:
+  1. vdst=v[0] is omitted because 0 is treated as "default"
+  2. src0 shows RawImm(257) instead of v[1]
+  3. User sees encoded values (257 = 256 + 1) instead of register names
+
+  Expected repr: VOP1(op=1, vdst=v[0], src0=v[1])
+  """
+  def test_repr_shows_registers_not_raw_imm(self):
+    inst = v_mov_b32_e32(v[0], v[1])
+    # Should show v[1], not RawImm(257)
+    self.assertNotIn("RawImm", repr(inst), "repr should not expose RawImm internal type")
+    self.assertIn("v[1]", repr(inst), "repr should show register name")
+
+  def test_repr_includes_zero_dst(self):
+    inst = v_mov_b32_e32(v[0], v[1])
+    # v[0] is a valid destination register, should be shown
+    self.assertIn("vdst", repr(inst), "repr should include vdst even when 0")
+
+  def test_repr_roundtrip(self):
+    # repr should produce something that can be eval'd back
+    inst = v_mov_b32_e32(v[0], v[1])
+    # This would require repr to output valid Python, e.g.:
+    # "VOP1(op=VOP1Op.V_MOV_B32, vdst=v[0], src0=v[1])"
+    r = repr(inst)
+    # At minimum, it should be human-readable
+    self.assertIn("v[", r, "repr should show register syntax")
+
+
+class TestInstructionEquality(unittest.TestCase):
+  """
+  Issue: No __eq__ method - instruction comparison requires repr() workaround.
+
+  Two identical instructions should compare equal with ==, but currently:
+    inst1 == inst2 returns False
+
+  The test_handwritten.py works around this with:
+    self.assertEqual(repr(self.inst), repr(reasm))
+  """
+  def test_identical_instructions_equal(self):
+    inst1 = v_mov_b32_e32(v[0], v[1])
+    inst2 = v_mov_b32_e32(v[0], v[1])
+    self.assertEqual(inst1, inst2, "identical instructions should be equal")
+
+  def test_different_instructions_not_equal(self):
+    inst1 = v_mov_b32_e32(v[0], v[1])
+    inst2 = v_mov_b32_e32(v[0], v[2])
+    self.assertNotEqual(inst1, inst2, "different instructions should not be equal")
+
+
+class TestVOPDHelperSignature(unittest.TestCase):
+  """
+  Issue: VOPD helper functions have confusing semantics.
+
+  v_dual_mul_f32 is defined as:
+    v_dual_mul_f32 = functools.partial(VOPD, VOPDOp.V_DUAL_MUL_F32)
+
+  This binds VOPDOp.V_DUAL_MUL_F32 to the FIRST positional arg of VOPD.__init__,
+  which is 'opx'. So v_dual_mul_f32 sets the X operation.
+
+  But then test_dual_mul in test_handwritten.py does:
+    v_dual_mul_f32(VOPDOp.V_DUAL_MUL_F32, vdstx=v[0], ...)
+
+  This passes V_DUAL_MUL_F32 as the SECOND positional arg (opy), making both
+  X and Y operations the same. This is confusing because:
+  1. The function name suggests it handles the X operation
+  2. But you still pass an opcode as the first arg (which becomes opy)
+
+  Expected: Either make the helper fully specify both ops, or make the
+  signature clearer about what the positional arg means.
+  """
+  def test_vopd_helper_opy_should_be_required(self):
+    # Using only keyword args "works" but opy silently defaults to 0
+    inst = v_dual_mul_f32(vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
+    self.assertEqual(inst.opx, VOPDOp.V_DUAL_MUL_F32)
+    # Bug: opy defaults to 0 (V_DUAL_FMAC_F32) silently - should require explicit opy
+    # This test documents the bug - it should fail once fixed
+    self.assertNotEqual(inst.opy, VOPDOp.V_DUAL_FMAC_F32, "opy should not silently default to FMAC")
+
+  def test_vopd_helper_positional_arg_is_opy(self):
+    # The first positional arg after the partial becomes opy, not a second opx
+    inst = v_dual_mul_f32(VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
+    self.assertEqual(inst.opx, VOPDOp.V_DUAL_MUL_F32)  # From partial
+    self.assertEqual(inst.opy, VOPDOp.V_DUAL_MOV_B32)  # From first positional arg
+
+
+class TestFieldAccessPreservesType(unittest.TestCase):
+  """
+  Issue: Field access loses type information.
+
+  After creating an instruction, accessing fields returns encoded int values:
+    inst = v_mov_b32_e32(v[0], v[1])
+    inst.vdst  # returns 0, not VGPR(0)
+
+  This makes it impossible to round-trip register types through field access.
+  """
+  def test_vdst_returns_register(self):
+    inst = v_mov_b32_e32(v[5], v[1])
+    vdst = inst.vdst
+    # Should return a VGPR, not an int
+    self.assertIsInstance(vdst, (VGPR, int), "vdst should return VGPR or at least be usable")
+    # Ideally: self.assertIsInstance(vdst, VGPR)
+
+  def test_src_returns_register_for_vgpr_source(self):
+    inst = v_mov_b32_e32(v[0], v[1])
+    # src0 is encoded as 257 (256 + 1 for v1)
+    # Ideally it should decode back to v[1]
+    src0_raw = inst._values.get('src0')
+    # Currently returns RawImm(257), should return VGPR(1) or similar
+    self.assertNotIsInstance(src0_raw, RawImm, "source should not be RawImm internally")
+
+
+class TestArgumentDiscoverability(unittest.TestCase):
+  """
+  Issue: No clear signature for positional arguments.
+
+  inspect.signature(s_load_b128) shows: (*args, literal=None, **kwargs)
+
+  Users have no way to know the argument order without reading source code.
+  The order is implicitly defined by the class field definition order.
+
+  Possible fixes:
+  1. Add explicit parameter names to functools.partial
+  2. Generate type stubs with proper signatures
+  3. Add docstrings listing the expected arguments
+  """
+  def test_signature_has_named_params(self):
+    import inspect
+    sig = inspect.signature(s_load_b128)
+    params = list(sig.parameters.keys())
+    # Currently: ['args', 'literal', 'kwargs'] (from *args, literal=None, **kwargs)
+    # Expected: something like ['sdata', 'sbase', 'soffset', 'offset', 'literal']
+    self.assertIn('sdata', params, "signature should show field names")
+
+
+class TestSpecialConstants(unittest.TestCase):
+  """
+  Issue: NULL and other constants are IntEnum values that might be confusing.
+
+  NULL = SrcEnum.NULL = 124, but users might expect NULL to be a special object
+  that clearly represents "no register" rather than a magic number.
+  """
+  def test_null_has_clear_repr(self):
+    # NULL should have a clear string representation
+    self.assertIn("NULL", str(NULL) or repr(NULL), "NULL should be clearly identifiable")
+
+  def test_null_is_distinguishable_from_int(self):
+    # NULL should be distinguishable from the raw integer 124
+    self.assertNotEqual(type(NULL), int, "NULL should not be plain int")
+
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/helpers.py

@@ -0,0 +1,24 @@
+"""Shared test helpers for RDNA3 tests."""
+import shutil
+from dataclasses import dataclass
+
+@dataclass
+class KernelInfo:
+  code: bytes
+  global_size: tuple[int, int, int]
+  local_size: tuple[int, int, int]
+  buf_idxs: list[int]  # indices into shared buffer pool
+  buf_sizes: list[int]  # sizes for each buffer index
+
+# LLVM tool detection (shared across test files)
+def get_llvm_mc():
+  """Find llvm-mc executable, preferring newer versions."""
+  for p in ['llvm-mc', 'llvm-mc-21', 'llvm-mc-20']:
+    if shutil.which(p): return p
+  raise FileNotFoundError("llvm-mc not found")
+
+def get_llvm_objdump():
+  """Find llvm-objdump executable, preferring newer versions."""
+  for p in ['llvm-objdump', 'llvm-objdump-21', 'llvm-objdump-20']:
+    if shutil.which(p): return p
+  raise FileNotFoundError("llvm-objdump not found")

extra/assembly/amd/test/test_compare_emulators.py

@@ -0,0 +1,398 @@
+# Test to compare Python and Rust RDNA3 emulators by running real tinygrad kernels
+import unittest, ctypes, os
+from dataclasses import dataclass
+from pathlib import Path
+
+# Set environment before any tinygrad imports to use MOCKGPU
+# This allows generating AMD GPU kernels without requiring real hardware
+os.environ["AMD"] = "1"
+os.environ["MOCKGPU"] = "1"
+os.environ["PYTHON_REMU"] = "1"
+
+from extra.assembly.amd.emu import WaveState, decode_program, step_wave, WAVE_SIZE, set_valid_mem_ranges
+from extra.assembly.amd.test.helpers import KernelInfo
+
+REMU_PATH = Path(__file__).parents[3] / "remu/target/release/libremu.so"
+
+def _is_f32_nan(bits: int) -> bool:
+  """Check if 32-bit value is a NaN (exponent all 1s, mantissa non-zero)."""
+  return (bits & 0x7f800000) == 0x7f800000 and (bits & 0x007fffff) != 0
+
+def _vals_equal(a: int, b: int) -> bool:
+  """Compare two 32-bit values, treating all NaN bit patterns as equal."""
+  if a == b: return True
+  return _is_f32_nan(a) and _is_f32_nan(b)
+
+@dataclass
+class StateSnapshot:
+  pc: int
+  scc: int
+  vcc: int
+  exec_mask: int
+  sgpr: list[int]
+  vgpr: list[list[int]]
+
+  def diff(self, other: 'StateSnapshot', n_lanes: int, arrow: str = " vs ") -> list[str]:
+    """Return list of differences between two states."""
+    diffs = []
+    if self.pc != other.pc: diffs.append(f"pc: {self.pc}{arrow}{other.pc}")
+    if self.scc != other.scc: diffs.append(f"scc: {self.scc}{arrow}{other.scc}")
+    if self.vcc != other.vcc: diffs.append(f"vcc: 0x{self.vcc:08x}{arrow}0x{other.vcc:08x}")
+    if self.exec_mask != other.exec_mask: diffs.append(f"exec: 0x{self.exec_mask:08x}{arrow}0x{other.exec_mask:08x}")
+    for i, (a, b) in enumerate(zip(self.sgpr, other.sgpr)):
+      # Skip VCC_LO/HI (106/107) and EXEC_LO/HI (126/127) as they alias vcc/exec_mask which are compared separately
+      if i in (106, 107, 126, 127): continue
+      if not _vals_equal(a, b): diffs.append(f"sgpr[{i}]: 0x{a:08x}{arrow}0x{b:08x}")
+    for lane in range(n_lanes):
+      for i, (a, b) in enumerate(zip(self.vgpr[lane], other.vgpr[lane])):
+        if not _vals_equal(a, b): diffs.append(f"vgpr[{lane}][{i}]: 0x{a:08x}{arrow}0x{b:08x}")
+    return diffs
+
+class CStateSnapshot(ctypes.Structure):
+  _fields_ = [("pc", ctypes.c_uint32), ("scc", ctypes.c_uint32), ("vcc", ctypes.c_uint32), ("exec_mask", ctypes.c_uint32),
+              ("sgpr", ctypes.c_uint32 * 128), ("vgpr", (ctypes.c_uint32 * 256) * 32)]
+
+  def to_snapshot(self) -> StateSnapshot:
+    return StateSnapshot(pc=self.pc, scc=self.scc, vcc=self.vcc, exec_mask=self.exec_mask,
+                         sgpr=list(self.sgpr), vgpr=[list(self.vgpr[i]) for i in range(32)])
+
+class RustEmulator:
+  def __init__(self):
+    self.lib = ctypes.CDLL(str(REMU_PATH))
+    self.lib.wave_create.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32]
+    self.lib.wave_create.restype = ctypes.c_void_p
+    self.lib.wave_step.argtypes = [ctypes.c_void_p]
+    self.lib.wave_step.restype = ctypes.c_int32
+    self.lib.wave_get_snapshot.argtypes = [ctypes.c_void_p, ctypes.POINTER(CStateSnapshot)]
+    self.lib.wave_set_sgpr.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32]
+    self.lib.wave_set_vgpr.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32]
+    self.lib.wave_init_lds.argtypes = [ctypes.c_void_p, ctypes.c_uint32]
+    self.lib.wave_free.argtypes = [ctypes.c_void_p]
+    self.ctx = None
+
+  def create(self, kernel: bytes, n_lanes: int):
+    kernel_buf = (ctypes.c_char * len(kernel)).from_buffer_copy(kernel)
+    self.ctx = self.lib.wave_create(ctypes.addressof(kernel_buf), len(kernel), n_lanes)
+    self._kernel_buf = kernel_buf
+
+  def step(self) -> int: return self.lib.wave_step(self.ctx)
+  def set_sgpr(self, idx: int, val: int): self.lib.wave_set_sgpr(self.ctx, idx, val)
+  def set_vgpr(self, lane: int, idx: int, val: int): self.lib.wave_set_vgpr(self.ctx, lane, idx, val)
+  def init_lds(self, size: int): self.lib.wave_init_lds(self.ctx, size)
+
+  def get_snapshot(self) -> StateSnapshot:
+    snap = CStateSnapshot()
+    self.lib.wave_get_snapshot(self.ctx, ctypes.byref(snap))
+    return snap.to_snapshot()
+
+  def free(self):
+    if self.ctx: self.lib.wave_free(self.ctx); self.ctx = None
+
+class PythonEmulator:
+  def __init__(self):
+    self.state: WaveState | None = None
+    self.program: dict | None = None
+    self.lds: bytearray | None = None
+    self.n_lanes = 0
+
+  def create(self, kernel: bytes, n_lanes: int):
+    self.program = decode_program(kernel)
+    self.state = WaveState()
+    self.state.exec_mask = (1 << n_lanes) - 1
+    self.lds = bytearray(65536)
+    self.n_lanes = n_lanes
+
+  def step(self) -> int:
+    assert self.program is not None and self.state is not None and self.lds is not None
+    return step_wave(self.program, self.state, self.lds, self.n_lanes)
+  def set_sgpr(self, idx: int, val: int):
+    assert self.state is not None
+    self.state.sgpr[idx]._val = val & 0xffffffff
+  def set_vgpr(self, lane: int, idx: int, val: int):
+    assert self.state is not None
+    self.state.vgpr[lane][idx]._val = val & 0xffffffff
+
+  def get_snapshot(self) -> StateSnapshot:
+    assert self.state is not None
+    return StateSnapshot(pc=self.state.pc, scc=self.state.scc, vcc=self.state.vcc & 0xffffffff,
+                         exec_mask=self.state.exec_mask & 0xffffffff, sgpr=[r._val for r in self.state.sgpr],
+                         vgpr=[[r._val for r in self.state.vgpr[i]] for i in range(WAVE_SIZE)])
+
+def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: tuple[int, int, int],
+                      program, max_steps: int, debug: bool, trace_len: int, kernel_idx: int = 0,
+                      max_workgroups: int = 8) -> tuple[bool, str, int]:
+  """Run a single kernel through both emulators. Returns (success, message, total_steps)."""
+  gx, gy, gz = global_size
+  total_steps = 0
+  wg_count = 0
+
+  for gidz in range(gz):
+    for gidy in range(gy):
+      for gidx in range(gx):
+        if wg_count >= max_workgroups: return True, f"Completed {wg_count} workgroups (limit reached)", total_steps
+        wg_count += 1
+        rust = RustEmulator()
+        python = PythonEmulator()
+        rust.create(kernel, n_lanes)
+        python.create(kernel, n_lanes)
+
+        # Initialize LDS (64KB, standard size for AMD GPUs)
+        rust.init_lds(65536)
+
+        for emu in (rust, python):
+          emu.set_sgpr(0, args_ptr & 0xffffffff)
+          emu.set_sgpr(1, (args_ptr >> 32) & 0xffffffff)
+          emu.set_sgpr(13, gidx)
+          emu.set_sgpr(14, gidy)
+          emu.set_sgpr(15, gidz)
+
+        step = 0
+        trace: list[tuple[int, int, str, StateSnapshot, StateSnapshot]] = []
+        try:
+          while step < max_steps:
+            rust_before = rust.get_snapshot()
+            python_before = python.get_snapshot()
+
+            inst = program.get(python_before.pc)
+            inst_str = inst.disasm() if inst else f"unknown at PC={python_before.pc}"
+            trace.append((step, python_before.pc, inst_str, rust_before, python_before))
+            if len(trace) > trace_len: trace.pop(0)
+
+            if debug: print(f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: PC={python_before.pc}, inst={inst_str}")
+
+            # Instructions with known Rust emulator bugs - sync Python to Rust after execution
+            # v_div_scale/v_div_fixup: Rust has different VCC handling
+            # v_cvt_f16_f32: Rust clears high 16 bits, but hardware (and Python) preserves them
+            sync_after = any(x in inst_str for x in ('v_div_scale_f32', 'v_div_scale_f64', 'v_div_fixup_f32', 'v_div_fixup_f64',
+                                                      'v_cvt_f16_f32'))
+            diffs = rust_before.diff(python_before, n_lanes)
+            if diffs:
+              trace_lines = []
+              for idx, (s, pc, d, rb, pb) in enumerate(trace):
+                trace_lines.append(f"    step {s}: PC={pc:3d} {d}")
+                if idx < len(trace) - 1:
+                  next_rb, next_pb = trace[idx + 1][3:5]
+                  rust_diffs = rb.diff(next_rb, n_lanes, "->")
+                  python_diffs = pb.diff(next_pb, n_lanes, "->")
+                  if rust_diffs: trace_lines.append(f"             rust:   {', '.join(rust_diffs[:5])}")
+                  if python_diffs: trace_lines.append(f"             python: {', '.join(python_diffs[:5])}")
+                  elif rust_diffs: trace_lines.append(f"             python: (no changes)")
+                else:
+                  # Last traced instruction - compare with current state
+                  rust_diffs = rb.diff(rust_before, n_lanes, "->")
+                  python_diffs = pb.diff(python_before, n_lanes, "->")
+                  if rust_diffs: trace_lines.append(f"             rust:   {', '.join(rust_diffs[:5])}")
+                  if python_diffs: trace_lines.append(f"             python: {', '.join(python_diffs[:5])}")
+                  elif rust_diffs: trace_lines.append(f"             python: (no changes)")
+              trace_str = "\n".join(trace_lines)
+              return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step} before inst '{inst_str}': states differ (rust vs python):\n  " + "\n  ".join(diffs[:10]) + f"\n  Recent instructions:\n{trace_str}", total_steps
+
+            rust_result = rust.step()
+            python_result = python.step()
+
+            if rust_result != python_result:
+              trace_str = "\n".join(f"    step {s}: PC={pc:3d} {d}" for s, pc, d, _, _ in trace)
+              return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: different return codes: rust={rust_result}, python={python_result}, inst={inst_str}\n  Recent instructions:\n{trace_str}", total_steps
+
+            # Sync Python state to Rust after instructions with known Rust emulator differences
+            if sync_after:
+              rust_after = rust.get_snapshot()
+              for i in range(128): python.set_sgpr(i, rust_after.sgpr[i])
+              for lane in range(n_lanes):
+                for i in range(256): python.set_vgpr(lane, i, rust_after.vgpr[lane][i])
+              assert python.state is not None
+              python.state.pc, python.state.scc, python.state.vcc, python.state.exec_mask = rust_after.pc, rust_after.scc, rust_after.vcc, rust_after.exec_mask
+
+            if rust_result == -1:
+              total_steps += step + 1
+              break
+            if rust_result == 1:
+              total_steps += step + 1
+              break
+            if rust_result < 0 and rust_result != -2:
+              return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: error code {rust_result}", total_steps
+
+            step += 1
+          else:
+            return False, f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Max steps ({max_steps}) reached", total_steps
+        finally:
+          rust.free()
+
+  return True, f"Completed {gx*gy*gz} workgroups", total_steps
+
+def compare_emulators_multi_kernel(kernels: list[KernelInfo], buf_pool: dict[int, int], max_steps: int = 1000,
+                                    debug: bool = False, trace_len: int = 10, buf_data: dict[int, bytes] | None = None) -> tuple[bool, str]:
+  """Run all kernels through both emulators with shared buffer pool."""
+  if buf_data is None: buf_data = {}
+
+  # Allocate shared buffer pool with padding for over-reads (GPU loads up to 16 bytes at once)
+  buf_id_to_ptr: dict[int, int] = {}
+  buffers = []
+  for buf_id, size in buf_pool.items():
+    padded_size = ((size + 15) // 16) * 16 + 16  # round up to 16 bytes + extra padding
+    # Initialize with data from COPY if available
+    init_data = buf_data.get(buf_id, b'\x00' * padded_size)
+    init_list = list(init_data) + [0] * (padded_size - len(init_data))
+    buf = (ctypes.c_uint8 * padded_size)(*init_list[:padded_size])
+    buffers.append((buf, padded_size))
+    buf_id_to_ptr[buf_id] = ctypes.addressof(buf)
+
+  # Set up valid memory ranges
+  ranges = {(ctypes.addressof(b), size) for b, size in buffers}
+
+  total_steps = 0
+  for ki, kernel in enumerate(kernels):
+    # Create args array for this kernel's buffers
+    args = (ctypes.c_uint64 * len(kernel.buf_idxs))(*[buf_id_to_ptr[bid] for bid in kernel.buf_idxs])
+    args_ptr = ctypes.addressof(args)
+
+    # Update valid ranges to include this args array
+    kernel_ranges = ranges | {(args_ptr, ctypes.sizeof(args))}
+    set_valid_mem_ranges(kernel_ranges)
+
+    program = decode_program(kernel.code)
+    n_lanes = kernel.local_size[0] * kernel.local_size[1] * kernel.local_size[2]
+
+    ok, msg, steps = run_single_kernel(
+      kernel.code, min(n_lanes, 32), args_ptr, kernel.global_size,
+      program, max_steps, debug, trace_len, ki
+    )
+    total_steps += steps
+    if not ok:
+      return False, msg
+
+  return True, f"Completed {len(kernels)} kernels, {total_steps} total steps"
+
+def compare_emulators_with_memory(kernel: bytes, n_lanes: int, buf_sizes: list, max_steps: int = 1000, debug: bool = False,
+                                   global_size: tuple[int, int, int] = (1, 1, 1), trace_len: int = 10) -> tuple[bool, str]:
+  """Run both emulators with memory set up for tinygrad kernels, executing all workgroups. Legacy wrapper."""
+  # Allocate buffers
+  buffers = []
+  for size in buf_sizes:
+    buf = (ctypes.c_uint8 * size)(*[0] * size)
+    buffers.append(buf)
+
+  # Create args array with buffer pointers
+  args = (ctypes.c_uint64 * len(buffers))(*[ctypes.addressof(b) for b in buffers])
+  args_ptr = ctypes.addressof(args)
+
+  # Set up valid memory ranges for Python emulator
+  ranges = {(ctypes.addressof(b), len(b)) for b in buffers}
+  ranges.add((args_ptr, ctypes.sizeof(args)))
+  set_valid_mem_ranges(ranges)
+
+  program = decode_program(kernel)
+  ok, msg, _ = run_single_kernel(kernel, n_lanes, args_ptr, global_size, program, max_steps, debug, trace_len)
+  return ok, msg
+
+def get_kernels_from_tinygrad(op_fn) -> tuple[list[KernelInfo], dict[int, int], dict[int, bytes]]:
+  """Compile a tinygrad operation and extract all kernels with their buffer mappings."""
+  from tinygrad import Tensor
+  from tinygrad.runtime.support.elf import elf_loader
+
+  out = op_fn(Tensor)
+  sched = out.schedule()
+  kernels = []
+  buf_pool: dict[int, int] = {}  # buffer id -> size
+  buf_data: dict[int, bytes] = {}  # buffer id -> initial data from COPY
+
+  for ei in sched:
+    lowered = ei.lower()
+    if ei.ast.op.name == 'COPY':
+      # Handle COPY: extract source data to initialize destination buffer
+      if len(lowered.bufs) >= 2:
+        dst_buf, src_buf = lowered.bufs[0], lowered.bufs[1]
+        dst_id = id(dst_buf)
+        if dst_id not in buf_pool:
+          buf_pool[dst_id] = dst_buf.nbytes
+        # Get source data if it's from numpy/CPU
+        if hasattr(src_buf, 'base') and src_buf.base is not None and hasattr(src_buf.base, '_buf'):
+          src_data = bytes(src_buf.base._buf)
+          buf_data[dst_id] = src_data
+    elif ei.ast.op.name == 'SINK':
+      if lowered.prg and lowered.prg.p.lib:
+        lib = bytes(lowered.prg.p.lib)
+        _, sections, _ = elf_loader(lib)
+        for sec in sections:
+          if sec.name == '.text':
+            buf_idxs = []
+            buf_sizes = []
+            for b in lowered.bufs:
+              buf_id = id(b)
+              if buf_id not in buf_pool:
+                buf_pool[buf_id] = b.nbytes
+              buf_idxs.append(buf_id)
+              buf_sizes.append(b.nbytes)
+            kernels.append(KernelInfo(
+              code=bytes(sec.content),
+              global_size=tuple(lowered.prg.p.global_size),
+              local_size=tuple(lowered.prg.p.local_size),
+              buf_idxs=buf_idxs,
+              buf_sizes=buf_sizes
+            ))
+  if not kernels: raise RuntimeError("No kernel found")
+  return kernels, buf_pool, buf_data
+
+def get_kernel_from_tinygrad(op_fn) -> tuple[bytes, tuple[int, int, int], tuple[int, int, int], list]:
+  """Compile a tinygrad operation and extract the last (main) kernel binary. Legacy wrapper."""
+  kernels, _, _ = get_kernels_from_tinygrad(op_fn)
+  k = kernels[-1]
+  return k.code, k.global_size, k.local_size, k.buf_sizes
+
+class TestTinygradKernels(unittest.TestCase):
+  """Compare emulators on real tinygrad-compiled kernels."""
+
+  def _test_kernel(self, op_fn, max_steps=10000):
+    kernels, buf_pool, buf_data = get_kernels_from_tinygrad(op_fn)
+    ok, msg = compare_emulators_multi_kernel(kernels, buf_pool, max_steps=max_steps, buf_data=buf_data)
+    self.assertTrue(ok, msg)
+
+  # Basic ops - consolidated tests covering key instruction patterns
+  def test_unary_ops(self): self._test_kernel(lambda T: T([-1.0, 0.0, 1.0, 2.0]).relu().exp().log().sqrt().reciprocal())
+  def test_binary_ops(self): self._test_kernel(lambda T: (T([1.0, 2.0]) + T([3.0, 4.0])) * T([0.5, 0.5]) - T([1.0, 1.0]))
+  def test_trig(self): self._test_kernel(lambda T: T([0.1, 1.0, 3.14, -1.0]*8).sin() + T([0.1, 1.0, 3.14, -1.0]*8).cos())
+  def test_compare(self): self._test_kernel(lambda T: (T.empty(64) < T.empty(64)).where(T.empty(64), T.empty(64)))
+  def test_bitwise(self): self._test_kernel(lambda T: (T([0xF0, 0x0F, 0xFF]*11).int() & T([0x0F, 0x0F, 0x00]*11).int()) | T([1]*33).int())
+  def test_int_ops(self): self._test_kernel(lambda T: ((T.empty(64).int() + T.empty(64).int()) * T.empty(64).int()).float())
+
+  # Reductions
+  def test_reduce(self): self._test_kernel(lambda T: T.empty(64).sum() + T.empty(64).max())
+  def test_argmax(self): self._test_kernel(lambda T: T.empty(64).argmax())
+
+  # Matmul
+  def test_gemm(self): self._test_kernel(lambda T: T.empty(8, 8) @ T.empty(8, 8), max_steps=100000)
+  def test_gemm_fp16(self): self._test_kernel(lambda T: T.empty(16, 16).half() @ T.empty(16, 16).half(), max_steps=100000)
+
+  # Complex ops
+  def test_softmax(self): self._test_kernel(lambda T: T.empty(16).softmax())
+  def test_layernorm(self): self._test_kernel(lambda T: T.empty(8, 8).layernorm())
+
+  # Memory patterns
+  def test_memory(self): self._test_kernel(lambda T: T.empty(4, 4).permute(1, 0).contiguous() + T.empty(4, 1).expand(4, 4))
+
+  # Cast ops
+  def test_cast(self): self._test_kernel(lambda T: T.empty(32).half().float() + T.empty(32).int().float())
+
+  # Pooling - regression for VCC wave32 mode
+  def test_pool2d(self): self._test_kernel(lambda T: T.empty(1, 1, 8, 8).avg_pool2d(kernel_size=(4,4)) + T.empty(1, 1, 8, 8).max_pool2d(kernel_size=(4,4)))
+
+  # Convolution
+  def test_conv2d(self): self._test_kernel(lambda T: T.empty(1, 2, 8, 8).conv2d(T.empty(2, 2, 3, 3)), max_steps=50000)
+
+  # Regression tests
+  def test_topk(self): self._test_kernel(lambda T: T.empty(64).topk(3)[0])
+  def test_interpolate(self): self._test_kernel(lambda T: T.empty(1,2,16,16).relu().cast('uint8').interpolate((8,8), mode="linear"))
+  def test_index_int64(self):
+    from tinygrad import dtypes
+    self._test_kernel(lambda T: T.empty(4, 4)[T.arange(4).cast(dtypes.int64), :])
+  def test_gelu(self): self._test_kernel(lambda T: T.empty(32, 32).gelu())
+  def test_cross_entropy(self):
+    import numpy as np
+    np.random.seed(0)
+    classes = np.random.randint(0, 10, (16,), dtype=np.int32).tolist()
+    x_np = np.random.randn(16, 10).astype(np.float32)
+    self._test_kernel(lambda T: (T(x_np.tolist()).reshape(16,10) + 0).cross_entropy((T(classes).int().reshape(16) + 0)))
+  def test_isinf(self): self._test_kernel(lambda T: T([float('-inf'), 0., float('inf'), 1.1]*8).isinf())
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_formats.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+"""Test MUBUF, MTBUF, MIMG, EXP, DS formats against LLVM."""
+import unittest
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.dsl import encode_src
+
+class TestMUBUF(unittest.TestCase):
+  """Test MUBUF (buffer) instructions."""
+
+  def test_buffer_load_b32_basic(self):
+    # buffer_load_b32 v5, off, s[8:11], s3 offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_load_b32_idxen(self):
+    # buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x50,0xe0,0x00,0x05,0x82,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, idxen=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x50,0xe0,0x00,0x05,0x82,0x03]))
+
+  def test_buffer_load_b32_offen(self):
+    # buffer_load_b32 v5, v0, s[8:11], s3 offen offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x50,0xe0,0x00,0x05,0x42,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, offen=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x50,0xe0,0x00,0x05,0x42,0x03]))
+
+  def test_buffer_load_b32_glc(self):
+    # buffer_load_b32 v5, off, s[8:11], s3 offset:4095 glc
+    # GFX11: encoding: [0xff,0x4f,0x50,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, glc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x4f,0x50,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_load_b32_slc(self):
+    # buffer_load_b32 v5, off, s[8:11], s3 offset:4095 slc
+    # GFX11: encoding: [0xff,0x1f,0x50,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, slc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x1f,0x50,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_load_b32_dlc(self):
+    # buffer_load_b32 v5, off, s[8:11], s3 offset:4095 dlc
+    # GFX11: encoding: [0xff,0x2f,0x50,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, dlc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x2f,0x50,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_load_b32_all_flags(self):
+    # buffer_load_b32 v5, off, s[8:11], s3 offset:4095 glc slc dlc
+    # GFX11: encoding: [0xff,0x7f,0x50,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, glc=1, slc=1, dlc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x7f,0x50,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_store_b32(self):
+    # buffer_store_b32 v1, off, s[12:15], s4 offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x68,0xe0,0x00,0x01,0x03,0x04]
+    inst = buffer_store_b32(vdata=v[1], vaddr=v[0], srsrc=s[12:16], soffset=s[4], offset=4095)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x68,0xe0,0x00,0x01,0x03,0x04]))
+
+  def test_buffer_load_b64(self):
+    # buffer_load_b64 v[5:6], off, s[8:11], s3 offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x54,0xe0,0x00,0x05,0x02,0x03]
+    inst = buffer_load_b64(vdata=v[5:7], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x54,0xe0,0x00,0x05,0x02,0x03]))
+
+  def test_buffer_load_soffset_m0(self):
+    # buffer_load_b32 v5, off, s[8:11], m0 offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x7d]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=M0, offset=4095)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x7d]))
+
+  def test_buffer_load_soffset_inline_const(self):
+    # buffer_load_b32 v5, off, s[8:11], 0 offset:4095
+    # GFX11: encoding: [0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x80]
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=0, offset=4095)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0x50,0xe0,0x00,0x05,0x02,0x80]))
+
+  def test_buffer_disasm_roundtrip(self):
+    inst = buffer_load_b32(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, glc=1)
+    decoded = MUBUF.from_bytes(inst.to_bytes())
+    self.assertEqual(decoded.to_bytes(), inst.to_bytes())
+
+
+class TestMTBUF(unittest.TestCase):
+  """Test MTBUF (typed buffer) instructions."""
+
+  def test_tbuffer_load_format_x(self):
+    # tbuffer_load_format_x v5, off, s[8:11], s3 format:[BUF_FMT_32_FLOAT] offset:4095
+    # BUF_FMT_32_FLOAT = 22
+    # GFX11: encoding: [0xff,0x0f,0xb0,0xe8,0x00,0x05,0x02,0x03]
+    inst = tbuffer_load_format_x(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, format=22)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0xb0,0xe8,0x00,0x05,0x02,0x03]))
+
+  def test_tbuffer_store_format_x(self):
+    # tbuffer_store_format_x v5, off, s[8:11], s3 format:[BUF_FMT_32_FLOAT] offset:4095
+    # BUF_FMT_32_FLOAT = 22
+    # GFX11: encoding: [0xff,0x0f,0xb2,0xe8,0x00,0x05,0x02,0x03]
+    inst = tbuffer_store_format_x(vdata=v[5], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, format=22)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x0f,0xb2,0xe8,0x00,0x05,0x02,0x03]))
+
+  def test_tbuffer_load_format_xy(self):
+    # tbuffer_load_format_xy v[5:6], off, s[8:11], s3 format:[BUF_FMT_32_32_FLOAT] offset:4095
+    # BUF_FMT_32_32_FLOAT = 50
+    # GFX11: encoding: [0xff,0x8f,0x90,0xe9,0x00,0x05,0x02,0x03]
+    inst = tbuffer_load_format_xy(vdata=v[5:7], vaddr=v[0], srsrc=s[8:12], soffset=s[3], offset=4095, format=50)
+    self.assertEqual(inst.to_bytes(), bytes([0xff,0x8f,0x90,0xe9,0x00,0x05,0x02,0x03]))
+
+
+class TestMIMG(unittest.TestCase):
+  """Test MIMG (image) instructions."""
+
+  def test_image_load_2d(self):
+    # image_load v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D
+    # GFX11: encoding: [0x04,0x0f,0x00,0xf0,0x04,0x00,0x00,0x00]
+    inst = image_load(vdata=v[0:4], vaddr=v[4:6], srsrc=s[0:8], dmask=0xf, dim=1)  # dim=1 is SQ_RSRC_IMG_2D
+    self.assertEqual(inst.to_bytes(), bytes([0x04,0x0f,0x00,0xf0,0x04,0x00,0x00,0x00]))
+
+  def test_image_store_2d(self):
+    # image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D
+    # GFX11: encoding: [0x04,0x0f,0x18,0xf0,0x04,0x00,0x00,0x00]
+    inst = image_store(vdata=v[0:4], vaddr=v[4:6], srsrc=s[0:8], dmask=0xf, dim=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x04,0x0f,0x18,0xf0,0x04,0x00,0x00,0x00]))
+
+  def test_image_load_1d(self):
+    # image_load v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+    # GFX11: encoding: [0x00,0x0f,0x00,0xf0,0x04,0x00,0x00,0x00]
+    inst = image_load(vdata=v[0:4], vaddr=v[4], srsrc=s[0:8], dmask=0xf, dim=0)  # dim=0 is SQ_RSRC_IMG_1D
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x0f,0x00,0xf0,0x04,0x00,0x00,0x00]))
+
+  def test_image_sample(self):
+    # image_sample v[0:3], v[4:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
+    # GFX11: encoding: [0x04,0x0f,0x6c,0xf0,0x04,0x00,0x00,0x08]
+    inst = image_sample(vdata=v[0:4], vaddr=v[4:6], srsrc=s[0:8], ssamp=s[8:12], dmask=0xf, dim=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x04,0x0f,0x6c,0xf0,0x04,0x00,0x00,0x08]))
+
+  def test_image_load_d16(self):
+    # image_load v[0:1], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D d16
+    # GFX11: encoding: [0x04,0x0f,0x02,0xf0,0x04,0x00,0x00,0x00]
+    inst = image_load(vdata=v[0:2], vaddr=v[4:6], srsrc=s[0:8], dmask=0xf, dim=1, d16=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x04,0x0f,0x02,0xf0,0x04,0x00,0x00,0x00]))
+
+
+class TestEXP(unittest.TestCase):
+  """Test EXP (export) instructions."""
+
+  def test_exp_mrt0(self):
+    # exp mrt0 v0, v1, v2, v3
+    # GFX11: encoding: [0x0f,0x00,0x00,0xf8,0x00,0x01,0x02,0x03]
+    inst = EXP(en=0xf, target=0, vsrc0=v[0], vsrc1=v[1], vsrc2=v[2], vsrc3=v[3])
+    self.assertEqual(inst.to_bytes(), bytes([0x0f,0x00,0x00,0xf8,0x00,0x01,0x02,0x03]))
+
+  def test_exp_mrtz(self):
+    # exp mrtz v4, v3, v2, v1
+    # GFX11: encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01]
+    inst = EXP(en=0xf, target=8, vsrc0=v[4], vsrc1=v[3], vsrc2=v[2], vsrc3=v[1])
+    self.assertEqual(inst.to_bytes(), bytes([0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01]))
+
+  def test_exp_mrtz_done(self):
+    # exp mrtz v4, v3, v2, v1 done
+    # GFX11: encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01]
+    inst = EXP(en=0xf, target=8, vsrc0=v[4], vsrc1=v[3], vsrc2=v[2], vsrc3=v[3], done=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x03]))
+
+  def test_exp_partial_mask(self):
+    # exp mrt0 v0, v1, off, off (en=0x3, only first two components)
+    # GFX11: encoding: [0x03,0x00,0x00,0xf8,0x00,0x01,0x00,0x00]
+    inst = EXP(en=0x3, target=0, vsrc0=v[0], vsrc1=v[1], vsrc2=v[0], vsrc3=v[0])
+    self.assertEqual(inst.to_bytes(), bytes([0x03,0x00,0x00,0xf8,0x00,0x01,0x00,0x00]))
+
+  def test_exp_row_en(self):
+    # exp mrtz v4, v3, v2, v1 row_en
+    # GFX11: encoding: [0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01]
+    inst = EXP(en=0xf, target=8, vsrc0=v[4], vsrc1=v[3], vsrc2=v[2], vsrc3=v[1], row=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01]))
+
+
+class TestDS(unittest.TestCase):
+  """Test DS (data share / LDS) instructions."""
+
+  def test_ds_store_b32(self):
+    # ds_store_b32 v0, v1
+    # GFX11: encoding: [0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]
+    inst = ds_store_b32(addr=v[0], data0=v[1])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]))
+
+  def test_ds_load_b32(self):
+    # ds_load_b32 v0, v1
+    # GFX11: encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x00]
+    inst = ds_load_b32(vdst=v[0], addr=v[1])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x00]))
+
+  def test_ds_store_b32_offset(self):
+    # ds_store_b32 v0, v1 offset:64
+    # GFX11: encoding: [0x40,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]
+    inst = ds_store_b32(addr=v[0], data0=v[1], offset0=64)
+    self.assertEqual(inst.to_bytes(), bytes([0x40,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]))
+
+  def test_ds_load_b64(self):
+    # ds_load_b64 v[0:1], v2
+    # GFX11: encoding: [0x00,0x00,0xd8,0xd9,0x02,0x00,0x00,0x00]
+    inst = ds_load_b64(vdst=v[0:2], addr=v[2])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0xd8,0xd9,0x02,0x00,0x00,0x00]))
+
+  def test_ds_add_u32(self):
+    # ds_add_u32 v0, v1
+    # GFX11: encoding: [0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00]
+    inst = ds_add_u32(addr=v[0], data0=v[1])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00]))
+
+  def test_ds_store_b32_gds(self):
+    # ds_store_b32 v0, v1 gds
+    # GFX11: encoding: [0x00,0x00,0x36,0xd8,0x00,0x01,0x00,0x00]
+    inst = ds_store_b32(addr=v[0], data0=v[1], gds=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x36,0xd8,0x00,0x01,0x00,0x00]))
+
+
+class TestVOP3(unittest.TestCase):
+  """Test VOP3 (3-operand vector) instructions."""
+
+  def test_v_fma_f32(self):
+    # v_fma_f32 v0, v1, v2, v3
+    # GFX11: encoding: [0x00,0x00,0x13,0xd6,0x01,0x05,0x0e,0x04]
+    inst = v_fma_f32(vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x13,0xd6,0x01,0x05,0x0e,0x04]))
+
+  def test_v_mad_f32(self):
+    # v_fmac_f32_e64 v0, v1, v2 (fmac is fma with implicit dst as src2)
+    # Use v_fma_f32 with vdst == src2
+    inst = v_fma_f32(vdst=v[0], src0=v[1], src1=v[2], src2=v[0])
+    self.assertEqual(inst.to_bytes()[:4], bytes([0x00,0x00,0x13,0xd6]))
+
+  def test_v_add3_u32(self):
+    # v_add3_u32 v0, v1, v2, v3
+    # GFX11: encoding: [0x00,0x00,0x55,0xd6,0x01,0x05,0x0e,0x04]
+    inst = v_add3_u32(vdst=v[0], src0=v[1], src1=v[2], src2=v[3])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x55,0xd6,0x01,0x05,0x0e,0x04]))
+
+
+class TestFLAT(unittest.TestCase):
+  """Test FLAT/GLOBAL/SCRATCH memory instructions."""
+
+  def test_global_load_b32(self):
+    # global_load_b32 v0, v[1:2], off (seg=2 for global)
+    # GFX11: encoding: [0x00,0x00,0x52,0xdc,0x01,0x00,0x7c,0x00]
+    inst = global_load_b32(vdst=v[0], addr=v[1:3], saddr=OFF)
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x52,0xdc,0x01,0x00,0x7c,0x00]))
+
+  def test_global_store_b32(self):
+    # global_store_b32 v[0:1], v2, off (seg=2 for global)
+    # GFX11: encoding: [0x00,0x00,0x6a,0xdc,0x00,0x02,0x7c,0x00]
+    inst = global_store_b32(addr=v[0:2], data=v[2], saddr=OFF)
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x6a,0xdc,0x00,0x02,0x7c,0x00]))
+
+  def test_global_load_b32_saddr(self):
+    # global_load_b32 v0, v1, s[0:1] (seg=2 for global)
+    # GFX11: encoding: [0x00,0x00,0x52,0xdc,0x01,0x00,0x00,0x00]
+    inst = global_load_b32(vdst=v[0], addr=v[1], saddr=s[0:2])
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x52,0xdc,0x01,0x00,0x00,0x00]))
+
+  def test_global_load_b32_offset(self):
+    # global_load_b32 v0, v[1:2], off offset:256 (seg=2 for global)
+    # GFX11: encoding: [0x00,0x01,0x52,0xdc,0x01,0x00,0x7c,0x00]
+    inst = global_load_b32(vdst=v[0], addr=v[1:3], saddr=OFF, offset=256)
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x01,0x52,0xdc,0x01,0x00,0x7c,0x00]))
+
+  def test_global_load_b64(self):
+    # global_load_b64 v[0:1], v[2:3], off (seg=2 for global)
+    # GFX11: encoding: [0x00,0x00,0x56,0xdc,0x02,0x00,0x7c,0x00]
+    inst = global_load_b64(vdst=v[0:2], addr=v[2:4], saddr=OFF)
+    self.assertEqual(inst.to_bytes(), bytes([0x00,0x00,0x56,0xdc,0x02,0x00,0x7c,0x00]))
+
+
+class TestSMEM(unittest.TestCase):
+  """Test SMEM (scalar memory) instructions - regression tests for glc/dlc bit positions."""
+
+  def test_smem_dlc_bit_position(self):
+    # s_load_b32 s5, s[2:3], s0 dlc - tests that DLC is at bit 13 (not bit 14)
+    # GFX11: encoding: [0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00]
+    inst = s_load_b32(sdata=s[5], sbase=s[2], soffset=s[0], dlc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00]))
+
+  def test_smem_glc_bit_position(self):
+    # s_load_b32 s5, s[2:3], s0 glc - tests that GLC is at bit 14 (not bit 16)
+    # GFX11: encoding: [0x41,0x41,0x00,0xf4,0x00,0x00,0x00,0x00]
+    inst = s_load_b32(sdata=s[5], sbase=s[2], soffset=s[0], glc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x41,0x41,0x00,0xf4,0x00,0x00,0x00,0x00]))
+
+  def test_smem_glc_dlc_combined(self):
+    # s_load_b32 s5, s[2:3], s0 glc dlc - tests both flags together
+    # GFX11: encoding: [0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00]
+    inst = s_load_b32(sdata=s[5], sbase=s[2], soffset=s[0], glc=1, dlc=1)
+    self.assertEqual(inst.to_bytes(), bytes([0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00]))
+
+  def test_smem_disasm_roundtrip_dlc(self):
+    # Test that disassembly/reassembly preserves DLC bit correctly
+    data = bytes([0x41,0x21,0x00,0xf4,0x00,0x00,0x00,0x00])
+    decoded = SMEM.from_bytes(data)
+    self.assertEqual(decoded.to_bytes(), data)
+
+  def test_smem_disasm_roundtrip_glc_dlc(self):
+    # Test that disassembly/reassembly preserves GLC+DLC bits correctly
+    data = bytes([0x41,0x61,0x00,0xf4,0x00,0x00,0x00,0x00])
+    decoded = SMEM.from_bytes(data)
+    self.assertEqual(decoded.to_bytes(), data)
+
+
+class TestVOP3Literal(unittest.TestCase):
+  """Test VOP3 literal handling - regression tests for Inst64 literal encoding."""
+
+  def test_vop3_with_literal(self):
+    # v_add3_u32 v5, vcc_hi, 0xaf123456, v255
+    # GFX11: encoding: [0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+    from extra.assembly.amd.dsl import RawImm
+    inst = VOP3(VOP3Op.V_ADD3_U32, vdst=v[5], src0=RawImm(107), src1=0xaf123456, src2=v[255])
+    expected = bytes([0x05,0x00,0x55,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf])
+    self.assertEqual(inst.to_bytes(), expected)
+
+  def test_vop3_literal_null_operand(self):
+    # v_add3_u32 v5, null, exec_lo, 0xaf123456
+    # GFX11: encoding: [0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf]
+    from extra.assembly.amd.dsl import RawImm
+    inst = VOP3(VOP3Op.V_ADD3_U32, vdst=v[5], src0=NULL, src1=RawImm(126), src2=0xaf123456)
+    expected = bytes([0x05,0x00,0x55,0xd6,0x7c,0xfc,0xfc,0x03,0x56,0x34,0x12,0xaf])
+    self.assertEqual(inst.to_bytes(), expected)
+
+  def test_vop3p_with_literal(self):
+    # Test VOP3P literal encoding (also uses Inst64)
+    from extra.assembly.amd.dsl import RawImm
+    inst = VOP3P(VOP3POp.V_PK_ADD_F16, vdst=v[5], src0=RawImm(240), src1=0x12345678, src2=v[0])
+    self.assertEqual(len(inst.to_bytes()), 12)  # 8 bytes + 4 byte literal
+
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_handwritten.py

@@ -0,0 +1,178 @@
+# do not change these tests. we need to fix bugs to make them pass
+# the Inst constructor should be looking at the types of the fields to correctly set the value
+
+import unittest, struct
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.dsl import Inst
+from extra.assembly.amd.asm import asm
+from extra.assembly.amd.test.test_roundtrip import compile_asm
+
+class TestIntegration(unittest.TestCase):
+  inst: Inst
+  def tearDown(self):
+    if not hasattr(self, 'inst'): return
+    b = self.inst.to_bytes()
+    st = self.inst.disasm()
+    reasm = asm(st)
+    desc = f"{st:25s} {self.inst} {b!r} {reasm}"
+    self.assertEqual(b, compile_asm(st), desc)
+    # TODO: this compare should work for valid things
+    #self.assertEqual(self.inst, reasm)
+    self.assertEqual(repr(self.inst), repr(reasm))
+    print(desc)
+
+  def test_load_b128(self):
+    self.inst = s_load_b128(s[4:7], s[0:1], NULL, 0)
+
+  def test_load_b128_wrong_size(self):
+    # this should have to be 4 regs on the loaded to
+    with self.assertRaises(Exception):
+      self.inst = s_load_b128(s[4:6], s[0:1], NULL, 0)
+
+  def test_mov_b32(self):
+    self.inst = s_mov_b32(s[80], s[0])
+
+  def test_mov_b64(self):
+    self.inst = s_mov_b64(s[80:81], s[0:1])
+
+  def test_mov_b32_wrong(self):
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b32(s[80:81], s[0:1])
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b32(s[80:81], s[0])
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b32(s[80], s[0:1])
+
+  def test_mov_b64_wrong(self):
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b64(s[80], s[0])
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b64(s[80], s[0:1])
+    with self.assertRaises(Exception):
+      self.inst = s_mov_b64(s[80:81], s[0])
+
+  def test_load_b128_no_0(self):
+    self.inst = s_load_b128(s[4:7], s[0:1], NULL)
+
+  def test_load_b128_s(self):
+    self.inst = s_load_b128(s[4:7], s[0:1], s[8], 0)
+
+  def test_load_b128_v(self):
+    with self.assertRaises(TypeError):
+      self.inst = s_load_b128(s[4:7], s[0:1], v[8], 0)
+
+  def test_load_b128_off(self):
+    self.inst = s_load_b128(s[4:7], s[0:1], NULL, 3)
+
+  def test_simple_stos(self):
+    self.inst = s_mov_b32(s[0], s[1])
+
+  def test_simple_wrong(self):
+    with self.assertRaises(TypeError):
+      self.inst = s_mov_b32(v[0], s[1])
+
+  def test_simple_vtov(self):
+    self.inst = v_mov_b32_e32(v[0], v[1])
+
+  def test_simple_stov(self):
+    self.inst = v_mov_b32_e32(v[0], s[2])
+
+  def test_simple_float_to_v(self):
+    self.inst = v_mov_b32_e32(v[0], 1.0)
+
+  def test_simple_v_to_float(self):
+    with self.assertRaises(TypeError):
+      self.inst = v_mov_b32_e32(1, v[0])
+
+  def test_simple_int_to_v(self):
+    self.inst = v_mov_b32_e32(v[0], 1)
+
+  def test_three_add(self):
+    self.inst = v_add_co_ci_u32_e32(v[3], s[7], v[3])
+
+  def test_three_add_v(self):
+    self.inst = v_add_co_ci_u32_e32(v[3], v[7], v[3])
+
+  def test_three_add_const(self):
+    self.inst = v_add_co_ci_u32_e32(v[3], 2.0, v[3])
+
+  def test_swaitcnt_lgkm(self): self.inst = s_waitcnt(0xfc07)
+  def test_swaitcnt_vm(self): self.inst = s_waitcnt(0x03f7)
+
+  def test_vmad(self):
+    self.inst = v_mad_u64_u32(v[1:2], NULL, s[2], 3, v[1:2])
+
+  def test_large_imm(self):
+    self.inst = v_mov_b32_e32(v[0], 0x1234)
+
+  def test_dual_mov(self):
+    self.inst = VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=v[2], srcy0=v[4])
+
+  def test_dual_mul(self):
+    self.inst = v_dual_mul_f32(VOPDOp.V_DUAL_MUL_F32, vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
+
+  def test_simple_int_to_s(self):
+    self.inst = s_mov_b32(s[0], 3)
+
+  def test_complex_int_to_s(self):
+    self.inst = s_mov_b32(s[0], 0x235646)
+
+  def test_simple_float_to_s(self):
+    self.inst = s_mov_b32(s[0], 1.0)
+
+  def test_complex_float_to_s(self):
+    self.inst = s_mov_b32(s[0], 1337.0)
+    int_inst = s_mov_b32(s[0], struct.unpack("I", struct.pack("f", 1337.0))[0])
+    self.assertEqual(self.inst, int_inst)
+
+class TestRegisterSliceSyntax(unittest.TestCase):
+  """
+  Issue: Register slice syntax should use AMD assembly convention (inclusive end).
+
+  In AMD assembly, s[4:7] means registers s4, s5, s6, s7 (4 registers, inclusive).
+  The DSL should match this convention so that:
+  - s[4:7] gives 4 registers
+  - Disassembler output can be copied directly back into DSL code
+
+  Fix: Change _RegFactory.__getitem__ to use inclusive end:
+    key.stop - key.start + 1  (instead of key.stop - key.start)
+  """
+  def test_register_slice_count(self):
+    # s[4:7] should give 4 registers: s4, s5, s6, s7 (AMD convention, inclusive)
+    reg = s[4:7]
+    self.assertEqual(reg.count, 4, "s[4:7] should give 4 registers (s4, s5, s6, s7)")
+
+  def test_register_slice_roundtrip(self):
+    # Round-trip: DSL -> disasm -> DSL should preserve register count
+    reg = s[4:7]  # 4 registers in AMD convention
+    inst = s_load_b128(reg, s[0:1], NULL, 0)
+    disasm = inst.disasm()
+    # Disasm shows s[4:7] - user should be able to copy this back
+    self.assertIn("s[4:7]", disasm)
+    # And s[4:7] in DSL should give the same 4 registers
+    reg_from_disasm = s[4:7]
+    self.assertEqual(reg_from_disasm.count, 4, "s[4:7] from disasm should give 4 registers")
+
+class TestInstructionEquality(unittest.TestCase):
+  """
+  Issue: No __eq__ method - instruction comparison requires repr() workaround.
+
+  Two identical instructions should compare equal with ==, but currently:
+    inst1 == inst2 returns False
+
+  The test_handwritten.py works around this with:
+    self.assertEqual(repr(self.inst), repr(reasm))
+  """
+  def test_identical_instructions_equal(self):
+    inst1 = v_mov_b32_e32(v[0], v[1])
+    inst2 = v_mov_b32_e32(v[0], v[1])
+    self.assertEqual(inst1, inst2, "identical instructions should be equal")
+
+  def test_different_instructions_not_equal(self):
+    inst1 = v_mov_b32_e32(v[0], v[1])
+    inst2 = v_mov_b32_e32(v[0], v[2])
+    self.assertNotEqual(inst1, inst2, "different instructions should not be equal")
+
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_integration.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+"""Integration test: round-trip RDNA3 assembly through AMD toolchain."""
+import unittest, re, io, sys, subprocess
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.asm import waitcnt, asm
+from extra.assembly.amd.test.helpers import get_llvm_mc
+
+def disassemble(lib: bytes, arch: str = "gfx1100") -> str:
+  """Disassemble ELF binary using tinygrad's compiler, return raw output."""
+  from tinygrad.runtime.support.compiler_amd import HIPCompiler
+  old_stdout = sys.stdout
+  sys.stdout = io.StringIO()
+  HIPCompiler(arch).disassemble(lib)
+  output = sys.stdout.getvalue()
+  sys.stdout = old_stdout
+  return output
+
+def parse_disassembly(raw: str) -> list[str]:
+  """Parse disassembly output to list of instruction mnemonics."""
+  lines = []
+  for line in raw.splitlines():
+    if line.startswith('\t'):
+      instr = line.split('//')[0].strip()
+      if instr: lines.append(instr)
+  return lines
+
+def assemble_and_disassemble(instructions: list, arch: str = "gfx1100") -> list[str]:
+  """Assemble instructions with our DSL, then disassemble with AMD toolchain."""
+  from tinygrad.runtime.support.compiler_amd import HIPCompiler
+
+  # Generate bytes from our DSL
+  code_bytes = b''.join(inst.to_bytes() for inst in instructions)
+
+  # Wrap in minimal ELF-compatible assembly with .byte directives
+  byte_str = ', '.join(f'0x{b:02x}' for b in code_bytes)
+  asm_src = f".text\n.globl test\n.p2align 8\n.type test,@function\ntest:\n.byte {byte_str}\n"
+
+  # Assemble with AMD COMGR and disassemble
+  lib = HIPCompiler(arch).compile(asm_src)
+  return parse_disassembly(disassemble(lib, arch))
+
+class TestIntegration(unittest.TestCase):
+  """Test our assembler output matches LLVM disassembly."""
+
+  def test_simple_sop1(self):
+    """Test SOP1 instructions round-trip."""
+    instructions = [
+      s_mov_b32(s[0], s[1]),
+      s_mov_b32(s[2], 0),
+      s_not_b32(s[3], s[4]),
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    self.assertIn('s_mov_b32', disasm[0])
+    self.assertIn('s_mov_b32', disasm[1])
+    self.assertIn('s_not_b32', disasm[2])
+
+  def test_simple_sop2(self):
+    """Test SOP2 instructions round-trip."""
+    instructions = [
+      s_add_u32(s[0], s[1], s[2]),
+      s_sub_u32(s[3], s[4], 10),
+      s_and_b32(s[5], s[6], s[7]),
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    self.assertIn('s_add_u32', disasm[0])
+    self.assertIn('s_sub_u32', disasm[1])
+    self.assertIn('s_and_b32', disasm[2])
+
+  def test_simple_vop2(self):
+    """Test VOP2 instructions round-trip."""
+    instructions = [
+      v_add_f32_e32(v[0], v[1], v[2]),
+      v_mul_f32_e32(v[3], 1.0, v[4]),  # 1.0 is inline constant
+      v_and_b32_e32(v[5], 10, v[6]),  # small inline constant
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    self.assertIn('v_add_f32', disasm[0])
+    self.assertIn('v_mul_f32', disasm[1])
+
+  def test_control_flow(self):
+    """Test control flow instructions."""
+    instructions = [
+      s_waitcnt(simm16=waitcnt(lgkmcnt=0)),
+      s_endpgm(),
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    self.assertIn('s_waitcnt', disasm[0])
+    self.assertIn('s_endpgm', disasm[1])
+
+  def test_memory_ops(self):
+    """Test memory instructions."""
+    instructions = [
+      s_load_b32(s[0], s[0:2], NULL),
+      s_waitcnt(simm16=waitcnt(lgkmcnt=0)),
+      global_store_b32(addr=v[0:2], data=v[2], saddr=OFF),
+      s_endpgm(),
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    self.assertIn('s_load_b32', disasm[0])
+    self.assertIn('s_waitcnt', disasm[1])
+    self.assertIn('global_store_b32', disasm[2])
+
+  def test_full_kernel(self):
+    """Test a complete kernel similar to tinygrad output."""
+    # Simple kernel: load value, add 1, store back
+    instructions = [
+      # Get thread ID
+      v_mov_b32_e32(v[0], s[0]),  # base addr low
+      v_mov_b32_e32(v[1], s[1]),  # base addr high
+      # Load value
+      global_load_b32(vdst=v[2], addr=v[0:2], saddr=OFF),
+      s_waitcnt(simm16=waitcnt(vmcnt=0)),
+      # Add 1.0
+      v_add_f32_e32(v[2], 1.0, v[2]),
+      # Store result
+      global_store_b32(addr=v[0:2], data=v[2], saddr=OFF),
+      s_endpgm(),
+    ]
+    disasm = assemble_and_disassemble(instructions)
+    # Verify key instructions are present
+    self.assertTrue(any('global_load' in d for d in disasm))
+    self.assertTrue(any('v_add_f32' in d for d in disasm))
+    self.assertTrue(any('global_store' in d for d in disasm))
+    self.assertTrue(any('s_endpgm' in d for d in disasm))
+
+  def test_bytes_roundtrip(self):
+    """Test that our bytes match what AMD assembler produces."""
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+
+    # Simple instruction
+    inst = s_mov_b32(s[0], s[1])
+    our_bytes = inst.to_bytes()
+
+    # Assemble same instruction with AMD toolchain
+    asm_src = ".text\n.globl test\n.p2align 8\n.type test,@function\ntest:\ns_mov_b32 s0, s1\n"
+    compiler = HIPCompiler("gfx1100")
+    lib = compiler.compile(asm_src)
+    raw = disassemble(lib)
+
+    for line in raw.splitlines():
+      if 's_mov_b32' in line and '//' in line:
+        # Extract hex bytes from comment: "// 000000001300: BE800001"
+        comment = line.split('//')[1].strip()
+        hex_str = comment.split(':')[1].strip()
+        # Convert big-endian hex string to little-endian bytes
+        amd_bytes = bytes.fromhex(hex_str)[::-1]  # reverse for little-endian
+        self.assertEqual(our_bytes, amd_bytes, f"Bytes mismatch: ours={our_bytes.hex()} AMD={amd_bytes.hex()}")
+        return
+    self.fail("Could not find s_mov_b32 in disassembly")
+
+class TestAsm(unittest.TestCase):
+  """Test asm() string parsing."""
+
+  def test_asm_basic(self):
+    """Test basic instruction parsing."""
+    inst = asm('s_mov_b32 s0, s1')
+    self.assertEqual(inst.to_bytes(), s_mov_b32(s[0], s[1]).to_bytes())
+
+  def test_asm_with_immediates(self):
+    """Test parsing with immediate values."""
+    inst = asm('s_add_u32 s0, s1, 10')
+    self.assertEqual(inst.to_bytes(), s_add_u32(s[0], s[1], 10).to_bytes())
+
+  def test_asm_float_const(self):
+    """Test parsing float constants."""
+    inst = asm('v_mul_f32_e32 v0, 1.0, v1')
+    self.assertEqual(inst.to_bytes(), v_mul_f32_e32(v[0], 1.0, v[1]).to_bytes())
+
+  def test_asm_hex_immediate(self):
+    """Test parsing hex immediates."""
+    inst = asm('s_waitcnt 0xfc07')
+    self.assertEqual(inst.to_bytes(), s_waitcnt(simm16=0xfc07).to_bytes())
+
+  def test_asm_special_regs(self):
+    """Test parsing special registers."""
+    inst = asm('s_mov_b32 s0, vcc_lo')
+    self.assertEqual(inst.to_bytes(), s_mov_b32(s[0], VCC_LO).to_bytes())
+
+  def test_asm_register_range(self):
+    """Test parsing register ranges."""
+    inst = asm('s_load_b128 s[4:7], s[0:1], null')
+    self.assertEqual(inst.to_bytes(), s_load_b128(s[4:7], s[0:1], NULL).to_bytes())
+
+  def test_asm_matches_llvm(self):
+    """Test asm() output matches LLVM assembler."""
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+    compiler = HIPCompiler('gfx1100')
+
+    def get_llvm_bytes(instr: str) -> bytes:
+      src = f'.text\n.globl test\n.p2align 8\n.type test,@function\ntest:\n{instr}\n'
+      lib = compiler.compile(src)
+      raw = disassemble(lib)
+      for line in raw.splitlines():
+        if instr.split()[0] in line and '//' in line:
+          hex_str = line.split('//')[1].strip().split(':')[1].strip()
+          return bytes.fromhex(hex_str)[::-1]
+      return b''
+
+    tests = ['s_mov_b32 s0, s1', 's_endpgm', 'v_add_f32_e32 v0, v1, v2']
+    for t in tests:
+      self.assertEqual(asm(t).to_bytes(), get_llvm_bytes(t), f"mismatch for: {t}")
+
+  def test_asm_vop3_modifiers(self):
+    """Test asm() with VOP3 modifiers (neg, abs, clamp)."""
+    def get_llvm_encoding(instr: str) -> str:
+      result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', '-mcpu=gfx1100', '-show-encoding'],
+                              input=instr, capture_output=True, text=True)
+      if m := re.search(r'encoding:\s*\[(.*?)\]', result.stdout):
+        return m.group(1).replace('0x','').replace(',','').replace(' ','')
+      return ''
+
+    tests = [
+      'v_fma_f32 v0, -v1, v2, v3',       # neg on src0
+      'v_fma_f32 v0, v1, |v2|, v3',      # abs on src1
+      'v_fma_f32 v0, v1, v2, v3 clamp',  # clamp
+      'v_fma_f32 v0, -v1, |v2|, v3 clamp',  # all modifiers
+      'v_fma_f32 v0, -|v1|, v2, v3',     # neg+abs on same operand
+    ]
+    for t in tests:
+      our_hex = asm(t).to_bytes().hex()
+      llvm_hex = get_llvm_encoding(t)
+      self.assertEqual(our_hex, llvm_hex, f"mismatch for: {t}")
+
+class TestTinygradIntegration(unittest.TestCase):
+  """Test that we can parse disassembled tinygrad kernels."""
+
+  def test_simple_add_kernel(self):
+    """Generate a simple add kernel from tinygrad and verify disassembly."""
+    from tinygrad import Tensor
+    from tinygrad.codegen import get_program
+    from tinygrad.renderer.cstyle import AMDHIPRenderer
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+    from tinygrad.uop.ops import Ops
+
+    # Create a computation that generates a real kernel
+    a = Tensor([1.0, 2.0, 3.0, 4.0]).realize()
+    b = Tensor([5.0, 6.0, 7.0, 8.0]).realize()
+    c = a + b
+
+    # Get schedule and find SINK
+    schedule = c.schedule()
+    sink_items = [si for si in schedule if si.ast.op == Ops.SINK]
+    self.assertTrue(len(sink_items) > 0, "No SINK in schedule")
+
+    # Generate program
+    renderer = AMDHIPRenderer('gfx1100')
+    prg = get_program(sink_items[0].ast, renderer)
+    self.assertIsNotNone(prg.src)
+
+    # Compile and disassemble
+    compiler = HIPCompiler('gfx1100')
+    lib = compiler.compile(prg.src)
+    raw_disasm = disassemble(lib)
+    instrs = parse_disassembly(raw_disasm)
+
+    # Verify we got some instructions
+    self.assertTrue(len(instrs) > 0, "No instructions in disassembly")
+    # Should have an endpgm
+    self.assertTrue(any('s_endpgm' in i for i in instrs), "Missing s_endpgm")
+
+  def test_matmul_kernel(self):
+    """Generate a matmul kernel and verify disassembly has expected patterns."""
+    from tinygrad import Tensor
+    from tinygrad.codegen import get_program
+    from tinygrad.renderer.cstyle import AMDHIPRenderer
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+    from tinygrad.uop.ops import Ops
+
+    # Create a small matmul
+    a = Tensor.rand(4, 4).realize()
+    b = Tensor.rand(4, 4).realize()
+    c = a @ b
+
+    # Get schedule
+    schedule = c.schedule()
+    sink_items = [si for si in schedule if si.ast.op == Ops.SINK]
+    self.assertTrue(len(sink_items) > 0)
+
+    # Generate and compile
+    renderer = AMDHIPRenderer('gfx1100')
+    prg = get_program(sink_items[0].ast, renderer)
+    compiler = HIPCompiler('gfx1100')
+    lib = compiler.compile(prg.src)
+    raw_disasm = disassemble(lib)
+    instrs = parse_disassembly(raw_disasm)
+
+    # Matmul should have multiply and add instructions
+    has_mul = any('mul' in i.lower() for i in instrs)
+    has_add = any('add' in i.lower() for i in instrs)
+    self.assertTrue(has_mul or has_add, "Matmul should have mul/add ops")
+
+  def test_disasm_to_bytes_roundtrip(self):
+    """Parse disassembled instructions and verify we can re-encode some of them."""
+    from tinygrad import Tensor
+    from tinygrad.codegen import get_program
+    from tinygrad.renderer.cstyle import AMDHIPRenderer
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+    from tinygrad.uop.ops import Ops
+
+    # Simple kernel
+    a = Tensor([1.0, 2.0, 3.0, 4.0]).realize()
+    b = (a * 2.0)
+
+    schedule = b.schedule()
+    sink_items = [si for si in schedule if si.ast.op == Ops.SINK]
+    if not sink_items: return  # skip if no kernel
+
+    renderer = AMDHIPRenderer('gfx1100')
+    prg = get_program(sink_items[0].ast, renderer)
+    compiler = HIPCompiler('gfx1100')
+    lib = compiler.compile(prg.src)
+    raw_disasm = disassemble(lib)
+
+    # Find s_endpgm and verify we can encode it
+    for line in raw_disasm.splitlines():
+      if 's_endpgm' in line and '//' in line:
+        # Extract bytes from comment
+        comment = line.split('//')[1].strip()
+        hex_str = comment.split(':')[1].strip()
+        amd_bytes = bytes.fromhex(hex_str)[::-1]
+
+        # Our encoding
+        our_inst = s_endpgm()
+        our_bytes = our_inst.to_bytes()
+
+        self.assertEqual(our_bytes, amd_bytes, f"s_endpgm mismatch: ours={our_bytes.hex()} AMD={amd_bytes.hex()}")
+        return
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_llvm.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Test RDNA3 assembler/disassembler against LLVM test vectors."""
+import unittest, re, subprocess
+from tinygrad.helpers import fetch
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.asm import asm
+from extra.assembly.amd.test.helpers import get_llvm_mc
+
+LLVM_BASE = "https://raw.githubusercontent.com/llvm/llvm-project/main/llvm/test/MC/AMDGPU"
+
+# Format info: (filename, format_class, op_enum)
+LLVM_TEST_FILES = {
+  # Scalar ALU
+  'sop1': ('gfx11_asm_sop1.s', SOP1, SOP1Op),
+  'sop2': ('gfx11_asm_sop2.s', SOP2, SOP2Op),
+  'sopp': ('gfx11_asm_sopp.s', SOPP, SOPPOp),
+  'sopk': ('gfx11_asm_sopk.s', SOPK, SOPKOp),
+  'sopc': ('gfx11_asm_sopc.s', SOPC, SOPCOp),
+  # Vector ALU
+  'vop1': ('gfx11_asm_vop1.s', VOP1, VOP1Op),
+  'vop2': ('gfx11_asm_vop2.s', VOP2, VOP2Op),
+  'vopc': ('gfx11_asm_vopc.s', VOPC, VOPCOp),
+  'vop3': ('gfx11_asm_vop3.s', VOP3, VOP3Op),
+  'vop3p': ('gfx11_asm_vop3p.s', VOP3P, VOP3POp),
+  'vop3sd': ('gfx11_asm_vop3.s', VOP3SD, VOP3SDOp),  # VOP3SD shares file with VOP3
+  'vinterp': ('gfx11_asm_vinterp.s', VINTERP, VINTERPOp),
+  'vopd': ('gfx11_asm_vopd.s', VOPD, VOPDOp),
+  'vopcx': ('gfx11_asm_vopcx.s', VOPC, VOPCOp),  # VOPCX uses VOPC format
+  # VOP3 promotions (VOP1/VOP2/VOPC promoted to VOP3 encoding)
+  'vop3_from_vop1': ('gfx11_asm_vop3_from_vop1.s', VOP3, VOP3Op),
+  'vop3_from_vop2': ('gfx11_asm_vop3_from_vop2.s', VOP3, VOP3Op),
+  'vop3_from_vopc': ('gfx11_asm_vop3_from_vopc.s', VOP3, VOP3Op),
+  'vop3_from_vopcx': ('gfx11_asm_vop3_from_vopcx.s', VOP3, VOP3Op),
+  # Memory
+  'ds': ('gfx11_asm_ds.s', DS, DSOp),
+  'smem': ('gfx11_asm_smem.s', SMEM, SMEMOp),
+  'flat': ('gfx11_asm_flat.s', FLAT, FLATOp),
+  'mubuf': ('gfx11_asm_mubuf.s', MUBUF, MUBUFOp),
+  'mtbuf': ('gfx11_asm_mtbuf.s', MTBUF, MTBUFOp),
+  'mimg': ('gfx11_asm_mimg.s', MIMG, MIMGOp),
+  # WMMA (matrix multiply)
+  'wmma': ('gfx11_asm_wmma.s', VOP3P, VOP3POp),
+  # Additional features
+  'vop3_features': ('gfx11_asm_vop3_features.s', VOP3, VOP3Op),
+  'vop3p_features': ('gfx11_asm_vop3p_features.s', VOP3P, VOP3POp),
+  'vopd_features': ('gfx11_asm_vopd_features.s', VOPD, VOPDOp),
+  # Alias files (alternative mnemonics)
+  'vop3_alias': ('gfx11_asm_vop3_alias.s', VOP3, VOP3Op),
+  'vop3p_alias': ('gfx11_asm_vop3p_alias.s', VOP3P, VOP3POp),
+  'vopc_alias': ('gfx11_asm_vopc_alias.s', VOPC, VOPCOp),
+  'vopcx_alias': ('gfx11_asm_vopcx_alias.s', VOPC, VOPCOp),
+  'vinterp_alias': ('gfx11_asm_vinterp_alias.s', VINTERP, VINTERPOp),
+  'smem_alias': ('gfx11_asm_smem_alias.s', SMEM, SMEMOp),
+  'mubuf_alias': ('gfx11_asm_mubuf_alias.s', MUBUF, MUBUFOp),
+  'mtbuf_alias': ('gfx11_asm_mtbuf_alias.s', MTBUF, MTBUFOp),
+}
+
+def parse_llvm_tests(text: str) -> list[tuple[str, bytes]]:
+  """Parse LLVM test format into (asm, expected_bytes) pairs."""
+  tests, lines = [], text.split('\n')
+  for i, line in enumerate(lines):
+    line = line.strip()
+    if not line or line.startswith(('//', '.', ';')): continue
+    asm_text = line.split('//')[0].strip()
+    if not asm_text: continue
+    for j in range(i, min(i + 3, len(lines))):
+      # Match GFX11, W32, or W64 encodings (all valid for gfx11)
+      if m := re.search(r'(?:GFX11|W32|W64)[^:]*:.*?encoding:\s*\[(.*?)\]', lines[j]):
+        hex_bytes = m.group(1).replace('0x', '').replace(',', '').replace(' ', '')
+        if hex_bytes:
+          try: tests.append((asm_text, bytes.fromhex(hex_bytes)))
+          except ValueError: pass
+        break
+  return tests
+
+def try_assemble(text: str):
+  """Try to assemble instruction text, return bytes or None on failure."""
+  try: return asm(text).to_bytes()
+  except: return None
+
+def compile_asm_batch(instrs: list[str]) -> list[bytes]:
+  """Compile multiple instructions with a single llvm-mc call."""
+  if not instrs: return []
+  asm_text = ".text\n" + "\n".join(instrs) + "\n"
+  result = subprocess.run(
+    [get_llvm_mc(), '-triple=amdgcn', '-mcpu=gfx1100', '-mattr=+real-true16,+wavefrontsize32', '-show-encoding'],
+    input=asm_text, capture_output=True, text=True, timeout=30)
+  if result.returncode != 0: raise RuntimeError(f"llvm-mc batch failed: {result.stderr.strip()}")
+  # Parse all encodings from output
+  results = []
+  for line in result.stdout.split('\n'):
+    if 'encoding:' not in line: continue
+    enc = line.split('encoding:')[1].strip()
+    if enc.startswith('[') and enc.endswith(']'):
+      results.append(bytes.fromhex(enc[1:-1].replace('0x', '').replace(',', '').replace(' ', '')))
+  if len(results) != len(instrs): raise RuntimeError(f"expected {len(instrs)} encodings, got {len(results)}")
+  return results
+
+class TestLLVM(unittest.TestCase):
+  """Test assembler and disassembler against all LLVM test vectors."""
+  tests: dict[str, list[tuple[str, bytes]]] = {}
+
+  @classmethod
+  def setUpClass(cls):
+    for name, (filename, _, _) in LLVM_TEST_FILES.items():
+      try:
+        data = fetch(f"{LLVM_BASE}/{filename}").read_bytes()
+        cls.tests[name] = parse_llvm_tests(data.decode('utf-8', errors='ignore'))
+      except Exception as e:
+        print(f"Warning: couldn't fetch {filename}: {e}")
+        cls.tests[name] = []
+
+# Generate test methods dynamically for each format
+def _make_asm_test(name):
+  def test(self):
+    passed, failed, skipped = 0, 0, 0
+    for asm_text, expected in self.tests.get(name, []):
+      result = try_assemble(asm_text)
+      if result is None: skipped += 1
+      elif result == expected: passed += 1
+      else: failed += 1
+    print(f"{name.upper()} asm: {passed} passed, {failed} failed, {skipped} skipped")
+    self.assertEqual(failed, 0)
+  return test
+
+def _make_disasm_test(name):
+  def test(self):
+    _, fmt_cls, op_enum = LLVM_TEST_FILES[name]
+    # VOP3SD opcodes that share encoding with VOP3 (only for vop3sd test, not vopc promotions)
+    vop3sd_opcodes = {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}
+    is_vopc_promotion = name in ('vop3_from_vopc', 'vop3_from_vopcx')
+    undocumented = {'smem': {34, 35}, 'sopk': {22, 23}, 'sopp': {8, 58, 59}}
+
+    # First pass: decode all instructions and collect disasm strings
+    to_test: list[tuple[str, bytes, str | None, str | None]] = []  # (asm_text, data, disasm_str, error)
+    skipped = 0
+    for asm_text, data in self.tests.get(name, []):
+      if len(data) > fmt_cls._size(): continue
+      temp_inst = fmt_cls.from_bytes(data)
+      temp_op = temp_inst._values.get('op', 0)
+      temp_op = temp_op.val if hasattr(temp_op, 'val') else temp_op
+      if temp_op in undocumented.get(name, set()): skipped += 1; continue
+      if name == 'sopp':
+        simm16 = temp_inst._values.get('simm16', 0)
+        simm16 = simm16.val if hasattr(simm16, 'val') else simm16
+        sopp_no_imm = {48, 54, 53, 55, 60, 61, 62}
+        if temp_op in sopp_no_imm and simm16 != 0: skipped += 1; continue
+      try:
+        if fmt_cls.__name__ in ('VOP3', 'VOP3SD'):
+          temp = VOP3.from_bytes(data)
+          op_val = temp._values.get('op', 0)
+          op_val = op_val.val if hasattr(op_val, 'val') else op_val
+          is_vop3sd = (op_val in vop3sd_opcodes) and not is_vopc_promotion
+          decoded = VOP3SD.from_bytes(data) if is_vop3sd else VOP3.from_bytes(data)
+          if is_vop3sd: VOP3SDOp(op_val)
+          else: VOP3Op(op_val)
+        else:
+          decoded = fmt_cls.from_bytes(data)
+          op_val = decoded._values.get('op', 0)
+          op_val = op_val.val if hasattr(op_val, 'val') else op_val
+          op_enum(op_val)
+        if decoded.to_bytes()[:len(data)] != data:
+          to_test.append((asm_text, data, None, "decode roundtrip failed"))
+          continue
+        to_test.append((asm_text, data, decoded.disasm(), None))
+      except Exception as e:
+        to_test.append((asm_text, data, None, f"exception: {e}"))
+
+    # Batch compile all disasm strings with single llvm-mc call
+    disasm_strs = [(i, t[2]) for i, t in enumerate(to_test) if t[2] is not None]
+    llvm_results = compile_asm_batch([s for _, s in disasm_strs]) if disasm_strs else []
+    llvm_map = {i: llvm_results[j] for j, (i, _) in enumerate(disasm_strs)}
+
+    # Match results back
+    passed, failed = 0, 0
+    failures: list[str] = []
+    for idx, (asm_text, data, disasm_str, error) in enumerate(to_test):
+      if error:
+        failed += 1; failures.append(f"{error} for {data.hex()}")
+      elif disasm_str is not None and idx in llvm_map:
+        llvm_bytes = llvm_map[idx]
+        if llvm_bytes is not None and llvm_bytes == data: passed += 1
+        elif llvm_bytes is not None: failed += 1; failures.append(f"'{disasm_str}': expected={data.hex()} got={llvm_bytes.hex()}")
+
+    print(f"{name.upper()} disasm: {passed} passed, {failed} failed" + (f", {skipped} skipped" if skipped else ""))
+    if failures[:10]: print("  " + "\n  ".join(failures[:10]))
+    self.assertEqual(failed, 0)
+  return test
+
+for name in LLVM_TEST_FILES:
+  setattr(TestLLVM, f'test_{name}_asm', _make_asm_test(name))
+  setattr(TestLLVM, f'test_{name}_disasm', _make_disasm_test(name))
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_mockgpu_invalid.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Test that invalid instructions raise exceptions through the mock GPU stack."""
+import unittest, subprocess, os, time
+
+class TestMockGPUInvalidInstruction(unittest.TestCase):
+  def test_unsupported_instruction_raises(self):
+    """Test that unsupported instructions raise immediately through the full MOCKGPU stack."""
+    test_code = '''
+import struct
+from tinygrad import Device, Tensor
+from tinygrad.engine.realize import get_runner
+from tinygrad.runtime.ops_amd import AMDProgram
+
+dev = Device["AMD"]
+a = Tensor([1.0]).realize()
+b = a + 1
+si = b.schedule()[-1]
+runner = get_runner(dev.device, si.ast)
+
+prg = runner._prg
+lib = bytearray(prg.lib)
+
+# Find s_endpgm (0xBFB00000) and replace with invalid SOPP op=127 (0xBFFF0000)
+found = False
+for i in range(0, len(lib) - 4, 4):
+  if struct.unpack("<I", lib[i:i+4])[0] == 0xBFB00000:
+    lib[i:i+4] = struct.pack("<I", 0xBFFF0000)
+    found = True
+    break
+assert found, "s_endpgm not found"
+
+patched_prg = AMDProgram(dev, "patched", bytes(lib))
+b.uop.buffer.allocate()
+patched_prg(b.uop.buffer._buf, a.uop.buffer._buf, global_size=(1,1,1), local_size=(1,1,1))
+dev.synchronize()
+'''
+
+    env = os.environ.copy()
+    env["AMD"] = "1"
+    env["MOCKGPU"] = "1"
+    env["PYTHON_REMU"] = "1"
+    env["HCQDEV_WAIT_TIMEOUT_MS"] = "10000"
+
+    st = time.perf_counter()
+    result = subprocess.run(["python", "-c", test_code], env=env, capture_output=True, text=True, timeout=60)
+    elapsed = time.perf_counter() - st
+
+    self.assertNotEqual(result.returncode, 0, "should have raised")
+    self.assertTrue("NotImplementedError" in result.stderr or "ValueError" in result.stderr,
+                    f"expected NotImplementedError or ValueError in stderr")
+    # Should exit immediately, not wait for the full timeout
+    self.assertLess(elapsed, 9.0, f"should exit immediately on emulator exception, took {elapsed:.1f}s")
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_pcode.py

@@ -0,0 +1,399 @@
+#!/usr/bin/env python3
+"""Tests for the RDNA3 pseudocode DSL."""
+import unittest
+from extra.assembly.amd.pcode import (Reg, TypedView, SliceProxy, ExecContext, compile_pseudocode, _expr, MASK32, MASK64,
+                                       _f32, _i32, _f16, _i16, f32_to_f16, _isnan, _bf16, _ibf16, bf16_to_f32, f32_to_bf16,
+                                       BYTE_PERMUTE, v_sad_u8, v_msad_u8)
+from extra.assembly.amd.autogen.rdna3.gen_pcode import _VOP3SDOp_V_DIV_SCALE_F32, _VOPCOp_V_CMP_CLASS_F32
+
+class TestReg(unittest.TestCase):
+  def test_u32_read(self):
+    r = Reg(0xDEADBEEF)
+    self.assertEqual(int(r.u32), 0xDEADBEEF)
+
+  def test_u32_write(self):
+    r = Reg(0)
+    r.u32 = 0x12345678
+    self.assertEqual(r._val, 0x12345678)
+
+  def test_f32_read(self):
+    r = Reg(0x40400000)  # 3.0f
+    self.assertAlmostEqual(float(r.f32), 3.0)
+
+  def test_f32_write(self):
+    r = Reg(0)
+    r.f32 = 3.0
+    self.assertEqual(r._val, 0x40400000)
+
+  def test_i32_signed(self):
+    r = Reg(0xFFFFFFFF)  # -1 as signed
+    self.assertEqual(int(r.i32), -1)
+
+  def test_u64(self):
+    r = Reg(0xDEADBEEFCAFEBABE)
+    self.assertEqual(int(r.u64), 0xDEADBEEFCAFEBABE)
+
+  def test_f64(self):
+    r = Reg(0x4008000000000000)  # 3.0 as f64
+    self.assertAlmostEqual(float(r.f64), 3.0)
+
+class TestTypedView(unittest.TestCase):
+  def test_bit_slice(self):
+    r = Reg(0xDEADBEEF)
+    # Slices return SliceProxy which supports .u32, .u16 etc (matching pseudocode like S1.u32[1:0].u32)
+    self.assertEqual(r.u32[7:0].u32, 0xEF)
+    self.assertEqual(r.u32[15:8].u32, 0xBE)
+    self.assertEqual(r.u32[23:16].u32, 0xAD)
+    self.assertEqual(r.u32[31:24].u32, 0xDE)
+    # Also works with int() for arithmetic
+    self.assertEqual(int(r.u32[7:0]), 0xEF)
+
+  def test_single_bit_read(self):
+    r = Reg(0b11010101)
+    self.assertEqual(r.u32[0], 1)
+    self.assertEqual(r.u32[1], 0)
+    self.assertEqual(r.u32[2], 1)
+    self.assertEqual(r.u32[3], 0)
+
+  def test_single_bit_write(self):
+    r = Reg(0)
+    r.u32[5] = 1
+    r.u32[3] = 1
+    self.assertEqual(r._val, 0b00101000)
+
+  def test_nested_bit_access(self):
+    # S0.u32[S1.u32[4:0]] - access bit at position from another register
+    s0 = Reg(0b11010101)
+    s1 = Reg(3)
+    bit_pos = s1.u32[4:0]  # SliceProxy, int value = 3
+    bit_val = s0.u32[int(bit_pos)]  # bit 3 of s0 = 0
+    self.assertEqual(int(bit_pos), 3)
+    self.assertEqual(bit_val, 0)
+
+  def test_arithmetic(self):
+    r1 = Reg(0x40400000)  # 3.0f
+    r2 = Reg(0x40800000)  # 4.0f
+    result = r1.f32 + r2.f32
+    self.assertAlmostEqual(result, 7.0)
+
+  def test_comparison(self):
+    r1 = Reg(5)
+    r2 = Reg(3)
+    self.assertTrue(r1.u32 > r2.u32)
+    self.assertFalse(r1.u32 < r2.u32)
+    self.assertTrue(r1.u32 != r2.u32)
+
+class TestSliceProxy(unittest.TestCase):
+  def test_slice_read(self):
+    r = Reg(0x56781234)
+    self.assertEqual(r[15:0].u16, 0x1234)
+    self.assertEqual(r[31:16].u16, 0x5678)
+
+  def test_slice_write(self):
+    r = Reg(0)
+    r[15:0].u16 = 0x1234
+    r[31:16].u16 = 0x5678
+    self.assertEqual(r._val, 0x56781234)
+
+  def test_slice_f16(self):
+    r = Reg(0)
+    r[15:0].f16 = 3.0
+    self.assertAlmostEqual(_f16(r._val & 0xffff), 3.0, places=2)
+
+class TestCompiler(unittest.TestCase):
+  def test_ternary(self):
+    result = _expr("a > b ? 1 : 0")
+    self.assertIn("if", result)
+    self.assertIn("else", result)
+
+  def test_type_prefix_strip(self):
+    self.assertEqual(_expr("1'0U"), "0")
+    self.assertEqual(_expr("32'1"), "1")
+    self.assertEqual(_expr("16'0xFFFF"), "0xFFFF")
+
+  def test_suffix_strip(self):
+    self.assertEqual(_expr("0ULL"), "0")
+    self.assertEqual(_expr("1LL"), "1")
+    self.assertEqual(_expr("5U"), "5")
+    self.assertEqual(_expr("3.14F"), "3.14")
+
+  def test_boolean_ops(self):
+    self.assertIn("and", _expr("a && b"))
+    self.assertIn("or", _expr("a || b"))
+    self.assertIn("!=", _expr("a <> b"))
+
+  def test_pack16(self):
+    result = _expr("{ a, b }")
+    self.assertIn("_pack", result)
+
+  def test_type_cast_strip(self):
+    self.assertEqual(_expr("64'U(x)"), "(x)")
+    self.assertEqual(_expr("32'I(y)"), "(y)")
+
+class TestExecContext(unittest.TestCase):
+  def test_float_add(self):
+    ctx = ExecContext(s0=0x40400000, s1=0x40800000)  # 3.0f, 4.0f
+    ctx.D0.f32 = ctx.S0.f32 + ctx.S1.f32
+    self.assertAlmostEqual(_f32(ctx.D0._val), 7.0)
+
+  def test_float_mul(self):
+    ctx = ExecContext(s0=0x40400000, s1=0x40800000)  # 3.0f, 4.0f
+    ctx.run("D0.f32 = S0.f32 * S1.f32")
+    self.assertAlmostEqual(_f32(ctx.D0._val), 12.0)
+
+  def test_scc_comparison(self):
+    ctx = ExecContext(s0=42, s1=42)
+    ctx.run("SCC = S0.u32 == S1.u32")
+    self.assertEqual(ctx.SCC._val, 1)
+
+  def test_scc_comparison_false(self):
+    ctx = ExecContext(s0=42, s1=43)
+    ctx.run("SCC = S0.u32 == S1.u32")
+    self.assertEqual(ctx.SCC._val, 0)
+
+  def test_ternary(self):
+    code = compile_pseudocode("D0.u32 = S0.u32 > S1.u32 ? 1'1U : 1'0U")
+    ctx = ExecContext(s0=5, s1=3)
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val, 1)
+
+  def test_pack(self):
+    code = compile_pseudocode("D0 = { S1[15:0].u16, S0[15:0].u16 }")
+    ctx = ExecContext(s0=0x1234, s1=0x5678)
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val, 0x56781234)
+
+  def test_tmp_with_typed_access(self):
+    code = compile_pseudocode("""tmp = S0.u32 + S1.u32
+D0.u32 = tmp.u32""")
+    ctx = ExecContext(s0=100, s1=200)
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val, 300)
+
+  def test_s_add_u32_pattern(self):
+    # Real pseudocode pattern from S_ADD_U32
+    code = compile_pseudocode("""tmp = 64'U(S0.u32) + 64'U(S1.u32)
+SCC = tmp >= 0x100000000ULL ? 1'1U : 1'0U
+D0.u32 = tmp.u32""")
+    # Test overflow case
+    ctx = ExecContext(s0=0xFFFFFFFF, s1=0x00000001)
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val, 0)  # Wraps to 0
+    self.assertEqual(ctx.SCC._val, 1)  # Carry set
+
+  def test_s_add_u32_no_overflow(self):
+    code = compile_pseudocode("""tmp = 64'U(S0.u32) + 64'U(S1.u32)
+SCC = tmp >= 0x100000000ULL ? 1'1U : 1'0U
+D0.u32 = tmp.u32""")
+    ctx = ExecContext(s0=100, s1=200)
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val, 300)
+    self.assertEqual(ctx.SCC._val, 0)  # No carry
+
+  def test_vcc_lane_read(self):
+    ctx = ExecContext(vcc=0b1010, lane=1)
+    # Lane 1 is set
+    self.assertEqual(ctx.VCC.u64[1], 1)
+    self.assertEqual(ctx.VCC.u64[2], 0)
+
+  def test_vcc_lane_write(self):
+    ctx = ExecContext(vcc=0, lane=0)
+    ctx.VCC.u64[3] = 1
+    ctx.VCC.u64[1] = 1
+    self.assertEqual(ctx.VCC._val, 0b1010)
+
+  def test_for_loop(self):
+    # CTZ pattern - find first set bit
+    code = compile_pseudocode("""tmp = -1
+for i in 0 : 31 do
+  if S0.u32[i] == 1 then
+    tmp = i
+D0.i32 = tmp""")
+    ctx = ExecContext(s0=0b1000)  # Bit 3 is set
+    ctx.run(code)
+    self.assertEqual(ctx.D0._val & MASK32, 3)
+
+  def test_result_dict(self):
+    ctx = ExecContext(s0=5, s1=3)
+    ctx.D0.u32 = 42
+    ctx.SCC._val = 1
+    result = ctx.result()
+    self.assertEqual(result['d0'], 42)
+    self.assertEqual(result['scc'], 1)
+
+class TestPseudocodeRegressions(unittest.TestCase):
+  """Regression tests for pseudocode instruction emulation bugs."""
+
+  def test_v_div_scale_f32_vcc_always_returned(self):
+    """V_DIV_SCALE_F32 must set VCC bit for the lane when scaling is needed.
+    The new calling convention uses Reg objects and modifies VCC in place."""
+    # Normal case: 1.0 / 3.0, no scaling needed, VCC should be 0
+    S0 = Reg(0x3f800000)  # 1.0
+    S1 = Reg(0x40400000)  # 3.0
+    S2 = Reg(0x3f800000)  # 1.0 (numerator)
+    D0 = Reg(0)
+    VCC = Reg(0)
+    _VOP3SDOp_V_DIV_SCALE_F32(S0, S1, S2, D0, Reg(0), VCC, 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    # VCC bit 0 should be 0 when no scaling needed
+    self.assertEqual(VCC._val & 1, 0, "VCC bit should be 0 when no scaling needed")
+
+  def test_v_cmp_class_f32_detects_quiet_nan(self):
+    """V_CMP_CLASS_F32 must correctly identify quiet NaN vs signaling NaN.
+    Bug: isQuietNAN and isSignalNAN both used math.isnan which can't distinguish them."""
+    quiet_nan = 0x7fc00000   # quiet NaN: exponent=255, bit22=1
+    signal_nan = 0x7f800001  # signaling NaN: exponent=255, bit22=0
+    # Test quiet NaN detection (bit 1 in mask)
+    s1_quiet = 0b0000000010  # bit 1 = quiet NaN
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(quiet_nan), Reg(s1_quiet), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 1, "Should detect quiet NaN with quiet NaN mask")
+    # Test signaling NaN detection (bit 0 in mask)
+    s1_signal = 0b0000000001  # bit 0 = signaling NaN
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(signal_nan), Reg(s1_signal), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 1, "Should detect signaling NaN with signaling NaN mask")
+    # Test that quiet NaN doesn't match signaling NaN mask
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(quiet_nan), Reg(s1_signal), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 0, "Quiet NaN should not match signaling NaN mask")
+    # Test that signaling NaN doesn't match quiet NaN mask
+    D0 = Reg(0)
+    _VOPCOp_V_CMP_CLASS_F32(Reg(signal_nan), Reg(s1_quiet), Reg(0), D0, Reg(0), Reg(0), 0, Reg(0xffffffff), Reg(0), None, Reg(0), Reg(0))
+    self.assertEqual(D0._val & 1, 0, "Signaling NaN should not match quiet NaN mask")
+
+  def test_isnan_with_typed_view(self):
+    """_isnan must work with TypedView objects, not just Python floats.
+    Bug: _isnan checked isinstance(x, float) which returned False for TypedView."""
+    nan_reg = Reg(0x7fc00000)  # quiet NaN
+    normal_reg = Reg(0x3f800000)  # 1.0
+    inf_reg = Reg(0x7f800000)  # +inf
+    self.assertTrue(_isnan(nan_reg.f32), "_isnan should return True for NaN TypedView")
+    self.assertFalse(_isnan(normal_reg.f32), "_isnan should return False for normal TypedView")
+    self.assertFalse(_isnan(inf_reg.f32), "_isnan should return False for inf TypedView")
+
+class TestBF16(unittest.TestCase):
+  """Tests for BF16 (bfloat16) support."""
+
+  def test_bf16_conversion(self):
+    """Test bf16 <-> f32 conversion."""
+    # bf16 is just the top 16 bits of f32
+    # 1.0f = 0x3f800000, bf16 = 0x3f80
+    self.assertAlmostEqual(_bf16(0x3f80), 1.0, places=2)
+    self.assertEqual(_ibf16(1.0), 0x3f80)
+    # 2.0f = 0x40000000, bf16 = 0x4000
+    self.assertAlmostEqual(_bf16(0x4000), 2.0, places=2)
+    self.assertEqual(_ibf16(2.0), 0x4000)
+    # -1.0f = 0xbf800000, bf16 = 0xbf80
+    self.assertAlmostEqual(_bf16(0xbf80), -1.0, places=2)
+    self.assertEqual(_ibf16(-1.0), 0xbf80)
+
+  def test_bf16_special_values(self):
+    """Test bf16 special values (inf, nan)."""
+    import math
+    # +inf: f32 = 0x7f800000, bf16 = 0x7f80
+    self.assertTrue(math.isinf(_bf16(0x7f80)))
+    self.assertEqual(_ibf16(float('inf')), 0x7f80)
+    # -inf: f32 = 0xff800000, bf16 = 0xff80
+    self.assertTrue(math.isinf(_bf16(0xff80)))
+    self.assertEqual(_ibf16(float('-inf')), 0xff80)
+    # NaN: quiet NaN bf16 = 0x7fc0
+    self.assertTrue(math.isnan(_bf16(0x7fc0)))
+    self.assertEqual(_ibf16(float('nan')), 0x7fc0)
+
+  def test_bf16_register_property(self):
+    """Test Reg.bf16 property."""
+    r = Reg(0)
+    r.bf16 = 3.0  # 3.0f = 0x40400000, bf16 = 0x4040
+    self.assertEqual(r._val & 0xffff, 0x4040)
+    self.assertAlmostEqual(float(r.bf16), 3.0, places=1)
+
+  def test_bf16_slice_property(self):
+    """Test SliceProxy.bf16 property."""
+    r = Reg(0x40404040)  # Two bf16 3.0 values
+    self.assertAlmostEqual(r[15:0].bf16, 3.0, places=1)
+    self.assertAlmostEqual(r[31:16].bf16, 3.0, places=1)
+
+class TestBytePermute(unittest.TestCase):
+  """Tests for BYTE_PERMUTE helper function (V_PERM_B32)."""
+
+  def test_byte_select_0_to_7(self):
+    """Test selecting bytes 0-7 from 64-bit data."""
+    # data = {s0, s1} where s0 is bytes 0-3, s1 is bytes 4-7
+    # Combined: 0x0706050403020100 (byte 0 = 0x00, byte 7 = 0x07)
+    data = 0x0706050403020100
+    for i in range(8):
+      self.assertEqual(BYTE_PERMUTE(data, i), i, f"byte {i} should be {i}")
+
+  def test_sign_extend_bytes(self):
+    """Test sign extension selectors 8-11."""
+    # sel 8: sign of byte 1 (bits 15:8)
+    # sel 9: sign of byte 3 (bits 31:24)
+    # sel 10: sign of byte 5 (bits 47:40)
+    # sel 11: sign of byte 7 (bits 63:56)
+    data = 0x8000800080008000  # All relevant bytes have sign bit set
+    self.assertEqual(BYTE_PERMUTE(data, 8), 0xff)
+    self.assertEqual(BYTE_PERMUTE(data, 9), 0xff)
+    self.assertEqual(BYTE_PERMUTE(data, 10), 0xff)
+    self.assertEqual(BYTE_PERMUTE(data, 11), 0xff)
+    data = 0x7f007f007f007f00  # No sign bits set
+    self.assertEqual(BYTE_PERMUTE(data, 8), 0x00)
+    self.assertEqual(BYTE_PERMUTE(data, 9), 0x00)
+    self.assertEqual(BYTE_PERMUTE(data, 10), 0x00)
+    self.assertEqual(BYTE_PERMUTE(data, 11), 0x00)
+
+  def test_constant_zero(self):
+    """Test selector 12 returns 0x00."""
+    self.assertEqual(BYTE_PERMUTE(0xffffffffffffffff, 12), 0x00)
+
+  def test_constant_ff(self):
+    """Test selectors >= 13 return 0xFF."""
+    for sel in [13, 14, 15, 255]:
+      self.assertEqual(BYTE_PERMUTE(0, sel), 0xff, f"sel {sel} should be 0xff")
+
+class TestSADHelpers(unittest.TestCase):
+  """Tests for V_SAD_U8 and V_MSAD_U8 helper functions."""
+
+  def test_v_sad_u8_basic(self):
+    """Test v_sad_u8 with simple values."""
+    # s0 = 0x04030201, s1 = 0x04030201 -> diff = 0 for all bytes
+    result = v_sad_u8(0x04030201, 0x04030201, 0)
+    self.assertEqual(result, 0)
+    # s0 = 0x05040302, s1 = 0x04030201 -> diff = 1+1+1+1 = 4
+    result = v_sad_u8(0x05040302, 0x04030201, 0)
+    self.assertEqual(result, 4)
+
+  def test_v_sad_u8_with_accumulator(self):
+    """Test v_sad_u8 with non-zero accumulator."""
+    # s0 = 0x05040302, s1 = 0x04030201, s2 = 100 -> 4 + 100 = 104
+    result = v_sad_u8(0x05040302, 0x04030201, 100)
+    self.assertEqual(result, 104)
+
+  def test_v_sad_u8_large_diff(self):
+    """Test v_sad_u8 with maximum byte differences."""
+    # s0 = 0xffffffff, s1 = 0x00000000 -> diff = 255*4 = 1020
+    result = v_sad_u8(0xffffffff, 0x00000000, 0)
+    self.assertEqual(result, 1020)
+
+  def test_v_msad_u8_basic(self):
+    """Test v_msad_u8 masks when reference byte is 0."""
+    # s0 = 0x10101010, s1 = 0x00000000 -> all masked, result = 0
+    result = v_msad_u8(0x10101010, 0x00000000, 0)
+    self.assertEqual(result, 0)
+    # s0 = 0x10101010, s1 = 0x01010101 -> diff = |0x10-0x01|*4 = 15*4 = 60
+    result = v_msad_u8(0x10101010, 0x01010101, 0)
+    self.assertEqual(result, 60)
+
+  def test_v_msad_u8_partial_mask(self):
+    """Test v_msad_u8 with partial masking."""
+    # s0 = 0x10101010, s1 = 0x00010001 -> bytes 1 and 3 masked
+    # diff = |0x10-0x01| + |0x10-0x01| = 15 + 15 = 30
+    result = v_msad_u8(0x10101010, 0x00010001, 0)
+    self.assertEqual(result, 30)
+
+  def test_v_msad_u8_with_accumulator(self):
+    """Test v_msad_u8 with non-zero accumulator."""
+    result = v_msad_u8(0x10101010, 0x01010101, 50)
+    self.assertEqual(result, 110)  # 60 + 50
+
+if __name__ == '__main__':
+  unittest.main()

extra/assembly/amd/test/test_pdf_parser.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Test that PDF parser correctly extracts format fields."""
+import unittest, os
+from extra.assembly.amd.autogen.rdna3 import (
+  SOP1, SOP2, SOPK, SOPP, VOP1, VOP2, VOP3SD, VOPC, FLAT, VOPD,
+  SOP1Op, SOP2Op, VOP1Op, VOP3Op
+)
+
+# expected formats with key fields and whether they have ENCODING
+EXPECTED_FORMATS = {
+  'DPP16': (['SRC0', 'DPP_CTRL', 'BANK_MASK', 'ROW_MASK'], False),
+  'DPP8': (['SRC0', 'LANE_SEL0', 'LANE_SEL7'], False),
+  'DS': (['OP', 'ADDR', 'DATA0', 'DATA1', 'VDST'], True),
+  'EXP': (['EN', 'TARGET', 'VSRC0', 'VSRC1', 'VSRC2', 'VSRC3'], True),
+  'FLAT': (['OP', 'ADDR', 'DATA', 'SADDR', 'VDST', 'OFFSET'], True),
+  'LDSDIR': (['VDST', 'OP'], True),
+  'MIMG': (['OP', 'VADDR', 'VDATA', 'SRSRC', 'DMASK'], True),
+  'MTBUF': (['OP', 'VADDR', 'VDATA', 'SRSRC', 'FORMAT', 'SOFFSET'], True),
+  'MUBUF': (['OP', 'VADDR', 'VDATA', 'SRSRC', 'SOFFSET'], True),
+  'SMEM': (['OP', 'SBASE', 'SDATA', 'OFFSET', 'SOFFSET'], True),
+  'SOP1': (['OP', 'SDST', 'SSRC0'], True),
+  'SOP2': (['OP', 'SDST', 'SSRC0', 'SSRC1'], True),
+  'SOPC': (['OP', 'SSRC0', 'SSRC1'], True),
+  'SOPK': (['OP', 'SDST', 'SIMM16'], True),
+  'SOPP': (['OP', 'SIMM16'], True),
+  'VINTERP': (['OP', 'VDST', 'SRC0', 'SRC1', 'SRC2'], True),
+  'VOP1': (['OP', 'VDST', 'SRC0'], True),
+  'VOP2': (['OP', 'VDST', 'SRC0', 'VSRC1'], True),
+  'VOP3': (['OP', 'VDST', 'SRC0', 'SRC1', 'SRC2'], True),
+  'VOP3P': (['OP', 'VDST', 'SRC0', 'SRC1', 'SRC2'], True),
+  'VOP3SD': (['OP', 'VDST', 'SDST', 'SRC0', 'SRC1', 'SRC2'], True),
+  'VOPC': (['OP', 'SRC0', 'VSRC1'], True),
+  'VOPD': (['OPX', 'OPY', 'SRCX0', 'SRCY0', 'VDSTX', 'VDSTY'], True),
+}
+
+# Skip PDF parsing tests by default - only run with TEST_PDF_PARSER=1
+# These are slow (~5s) and only needed when regenerating autogen/
+@unittest.skipUnless(os.environ.get("TEST_PDF_PARSER"), "set TEST_PDF_PARSER=1 to run PDF parser tests")
+class TestPDFParserGenerate(unittest.TestCase):
+  """Test the PDF parser by running generate() and checking results."""
+
+  def test_pdf_parser(self):
+    """Single test that validates all PDF parser outputs."""
+    from extra.assembly.amd.dsl import generate
+    result = generate()
+
+    # test_all_formats_present
+    for fmt_name in EXPECTED_FORMATS:
+      self.assertIn(fmt_name, result["formats"], f"missing format {fmt_name}")
+
+    # test_format_count
+    self.assertEqual(len(result["formats"]), 23)
+
+    # test_no_duplicate_fields
+    for fmt_name, fields in result["formats"].items():
+      field_names = [f[0] for f in fields]
+      self.assertEqual(len(field_names), len(set(field_names)), f"{fmt_name} has duplicate fields: {field_names}")
+
+    # test_expected_fields
+    for fmt_name, (expected_fields, has_encoding) in EXPECTED_FORMATS.items():
+      fields = {f[0] for f in result["formats"].get(fmt_name, [])}
+      for field in expected_fields:
+        self.assertIn(field, fields, f"{fmt_name} missing {field}")
+      if has_encoding:
+        self.assertIn("ENCODING", fields, f"{fmt_name} should have ENCODING")
+      else:
+        self.assertNotIn("ENCODING", fields, f"{fmt_name} should not have ENCODING")
+
+    # test_vopd_no_dpp16_fields
+    vopd_fields = {f[0] for f in result["formats"].get("VOPD", [])}
+    for field in ['DPP_CTRL', 'BANK_MASK', 'ROW_MASK']:
+      self.assertNotIn(field, vopd_fields, f"VOPD should not have {field}")
+
+    # test_dpp16_no_vinterp_fields
+    dpp16_fields = {f[0] for f in result["formats"].get("DPP16", [])}
+    for field in ['VDST', 'WAITEXP']:
+      self.assertNotIn(field, dpp16_fields, f"DPP16 should not have {field}")
+
+    # test_sopp_no_smem_fields
+    sopp_fields = {f[0] for f in result["formats"].get("SOPP", [])}
+    for field in ['SBASE', 'SDATA']:
+      self.assertNotIn(field, sopp_fields, f"SOPP should not have {field}")
+
+class TestPDFParser(unittest.TestCase):
+  """Verify format classes have correct fields from PDF parsing."""
+
+  def test_sop2_fields(self):
+    """SOP2 should have op, sdst, ssrc0, ssrc1."""
+    for field in ['op', 'sdst', 'ssrc0', 'ssrc1']:
+      self.assertIn(field, SOP2._fields)
+    self.assertEqual(SOP2._fields['op'].hi, 29)
+    self.assertEqual(SOP2._fields['op'].lo, 23)
+
+  def test_sop1_fields(self):
+    """SOP1 should have op, sdst, ssrc0 with correct bit positions."""
+    for field in ['op', 'sdst', 'ssrc0']:
+      self.assertIn(field, SOP1._fields)
+    self.assertNotIn('simm16', SOP1._fields)
+    self.assertEqual(SOP1._fields['ssrc0'].hi, 7)
+    self.assertEqual(SOP1._fields['ssrc0'].lo, 0)
+    assert SOP1._encoding is not None
+    self.assertEqual(SOP1._encoding[0].hi, 31)
+    self.assertEqual(SOP1._encoding[1], 0b101111101)
+
+  def test_vop3sd_fields(self):
+    """VOP3SD should have all fields including src0/src1/src2 from page continuation."""
+    for field in ['op', 'vdst', 'sdst', 'src0', 'src1', 'src2']:
+      self.assertIn(field, VOP3SD._fields)
+    self.assertEqual(VOP3SD._fields['src0'].hi, 40)
+    self.assertEqual(VOP3SD._fields['src0'].lo, 32)
+    self.assertEqual(VOP3SD._size(), 8)
+
+  def test_flat_has_vdst(self):
+    """FLAT should have vdst field."""
+    self.assertIn('vdst', FLAT._fields)
+    self.assertEqual(FLAT._fields['vdst'].hi, 63)
+    self.assertEqual(FLAT._fields['vdst'].lo, 56)
+
+  def test_encoding_bits(self):
+    """Verify encoding bits are correct for major formats."""
+    tests = [
+      (SOP2, 31, 30, 0b10),
+      (SOPK, 31, 28, 0b1011),
+      (SOPP, 31, 23, 0b101111111),
+      (VOP1, 31, 25, 0b0111111),
+      (VOP2, 31, 31, 0b0),
+      (VOPC, 31, 25, 0b0111110),
+      (FLAT, 31, 26, 0b110111),
+    ]
+    for cls, hi, lo, val in tests:
+      assert cls._encoding is not None
+      self.assertEqual(cls._encoding[0].hi, hi, f"{cls.__name__} encoding hi")
+      self.assertEqual(cls._encoding[0].lo, lo, f"{cls.__name__} encoding lo")
+      self.assertEqual(cls._encoding[1], val, f"{cls.__name__} encoding val")
+
+  def test_opcode_enums_exist(self):
+    """Verify opcode enums are generated with expected counts."""
+    self.assertGreater(len(SOP1Op), 50)
+    self.assertGreater(len(SOP2Op), 50)
+    self.assertGreater(len(VOP1Op), 50)
+    self.assertGreater(len(VOP3Op), 200)
+
+  def test_vopd_no_duplicate_fields(self):
+    """VOPD should not have duplicate fields and should not include DPP16 fields."""
+    field_names = list(VOPD._fields.keys())
+    self.assertEqual(len(field_names), len(set(field_names)))
+    for field in ['srcx0', 'srcy0', 'opx', 'opy']:
+      self.assertIn(field, VOPD._fields)
+    for field in ['dpp_ctrl', 'bank_mask', 'row_mask']:
+      self.assertNotIn(field, VOPD._fields)
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_rdna3_asm.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import unittest, subprocess
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.test.helpers import get_llvm_mc
+
+def llvm_assemble(asm: str) -> bytes:
+  """Assemble using llvm-mc and return bytes."""
+  result = subprocess.run(
+    [get_llvm_mc(), "-triple=amdgcn", "-mcpu=gfx1100", "-show-encoding"],
+    input=asm, capture_output=True, text=True
+  )
+  out = b''
+  for line in result.stdout.split('\n'):
+    if 'encoding:' in line:
+      enc = line.split('encoding:')[1].strip()
+      enc = enc.strip('[]').replace('0x', '').replace(',', '')
+      out += bytes.fromhex(enc)
+  if not out: raise ValueError(f"no encoding found: {result.stdout} {result.stderr}")
+  return out
+
+class TestRDNA3Asm(unittest.TestCase):
+  def test_full_program(self):
+    """Test the full program from rdna3fun.py matches llvm-mc output."""
+    program = [
+      v_bfe_u32(v[1], v[0], 10, 10),
+      s_load_b128(s[4:7], s[0:1], NULL),
+      v_and_b32_e32(v[0], 0x3FF, v[0]),
+      s_mulk_i32(s[3], 0x87),
+      v_mad_u64_u32(v[1:2], NULL, s[2], 3, v[1:2]),
+      v_mul_u32_u24_e32(v[0], 45, v[0]),
+      v_ashrrev_i32_e32(v[2], 31, v[1]),
+      v_add3_u32(v[0], v[0], s[3], v[1]),
+      v_lshlrev_b64(v[2:3], 2, v[1:2]),
+      v_ashrrev_i32_e32(v[1], 31, v[0]),
+      v_lshlrev_b64(v[0:1], 2, v[0:1]),
+      s_waitcnt(0xfc07),  # lgkmcnt(0)
+      v_add_co_u32(v[2], VCC_LO, s[6], v[2]),
+      v_add_co_ci_u32_e32(v[3], s[7], v[3]),
+      v_add_co_u32(v[0], VCC_LO, s[4], v[0]),
+      global_load_b32(vdst=v[2], addr=v[2], saddr=OFF),
+      v_add_co_ci_u32_e32(v[1], s[5], v[1]),
+      s_waitcnt(0x03f7),  # vmcnt(0)
+      global_store_b32(addr=v[0], data=v[2], saddr=OFF),
+      s_endpgm(),
+    ]
+
+    asm = """
+v_bfe_u32 v1, v0, 10, 10
+s_load_b128 s[4:7], s[0:1], null
+v_and_b32_e32 v0, 0x3FF, v0
+s_mulk_i32 s3, 0x87
+v_mad_u64_u32 v[1:2], null, s2, 3, v[1:2]
+v_mul_u32_u24_e32 v0, 45, v0
+v_ashrrev_i32_e32 v2, 31, v1
+v_add3_u32 v0, v0, s3, v1
+v_lshlrev_b64 v[2:3], 2, v[1:2]
+v_ashrrev_i32_e32 v1, 31, v0
+v_lshlrev_b64 v[0:1], 2, v[0:1]
+s_waitcnt lgkmcnt(0)
+v_add_co_u32 v2, vcc_lo, s6, v2
+v_add_co_ci_u32_e32 v3, vcc_lo, s7, v3, vcc_lo
+v_add_co_u32 v0, vcc_lo, s4, v0
+global_load_b32 v2, v[2:3], off
+v_add_co_ci_u32_e32 v1, vcc_lo, s5, v1, vcc_lo
+s_waitcnt vmcnt(0)
+global_store_b32 v[0:1], v2, off
+s_endpgm
+"""
+    expected = llvm_assemble(asm)
+    for inst,rt in zip(program, asm.strip().split("\n")): print(f"{inst.disasm():50s} {rt}")
+    actual = b''.join(inst.to_bytes() for inst in program)
+    self.assertEqual(actual, expected)
+
+  def test_sop2_s_add_u32(self):
+    inst = SOP2(SOP2Op.S_ADD_U32, s[3], s[0], s[1])
+    expected = llvm_assemble("s_add_u32 s3, s0, s1")
+    self.assertEqual(inst.to_bytes(), expected)
+
+  def test_vop2_v_and_b32_inline_const(self):
+    inst = v_and_b32_e32(v[0], 10, v[0])
+    expected = llvm_assemble("v_and_b32_e32 v0, 10, v0")
+    self.assertEqual(inst.to_bytes(), expected)
+
+  def test_sopp_s_endpgm(self):
+    inst = s_endpgm()
+    expected = llvm_assemble("s_endpgm")
+    self.assertEqual(inst.to_bytes(), expected)
+
+  def test_sop1_s_mov_b32(self):
+    inst = s_mov_b32(s[0], s[1])
+    expected = llvm_assemble("s_mov_b32 s0, s1")
+    self.assertEqual(inst.to_bytes(), expected)
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_roundtrip.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+"""Roundtrip tests: generate tinygrad kernels, decode instructions, re-encode, verify match."""
+import unittest, io, sys, re, subprocess, os
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.dsl import Inst
+from extra.assembly.amd.asm import asm
+from extra.assembly.amd.test.helpers import get_llvm_mc, get_llvm_objdump
+
+# Instruction format detection based on encoding bits
+def detect_format(data: bytes) -> type[Inst] | None:
+  """Detect instruction format from machine code bytes."""
+  if len(data) < 4: return None
+  word = int.from_bytes(data[:4], 'little')
+  enc_9bit = (word >> 23) & 0x1FF  # 9-bit encoding for SOP1/SOPC/SOPP
+  enc_8bit = (word >> 24) & 0xFF
+
+  # Check 9-bit encodings first (most specific)
+  if enc_9bit == 0x17D: return SOP1  # bits 31:23 = 101111101
+  if enc_9bit == 0x17E: return SOPC  # bits 31:23 = 101111110
+  if enc_9bit == 0x17F: return SOPP  # bits 31:23 = 101111111
+  # SOPK: bits 31:28 = 1011, bits 27:23 = opcode (check after SOP1/SOPC/SOPP)
+  if enc_8bit in range(0xB0, 0xC0): return SOPK
+  # SOP2: bits 31:23 in range 0x100-0x17C (0x80-0xBE in bits 31:24, but not SOPK)
+  if 0x80 <= enc_8bit <= 0x9F: return SOP2
+  # VOP1: bits 31:25 = 0111111 (0x3F)
+  if (word >> 25) == 0x3F: return VOP1
+  # VOPC: bits 31:25 = 0111110 (0x3E)
+  if (word >> 25) == 0x3E: return VOPC
+  # VOP2: bits 31:30 = 00
+  if (word >> 30) == 0: return VOP2
+
+  # Check 64-bit formats
+  if len(data) >= 8:
+    if enc_8bit in (0xD4, 0xD5, 0xD7): return VOP3
+    if enc_8bit == 0xD6: return VOP3SD
+    if enc_8bit == 0xCC: return VOP3P
+    if enc_8bit == 0xCD: return VINTERP
+    if enc_8bit in (0xC8, 0xC9): return VOPD
+    if enc_8bit == 0xF4: return SMEM
+    if enc_8bit == 0xD8: return DS
+    if enc_8bit in (0xDC, 0xDD, 0xDE, 0xDF): return FLAT
+    if enc_8bit in (0xE0, 0xE1, 0xE2, 0xE3): return MUBUF
+    if enc_8bit in (0xE8, 0xE9, 0xEA, 0xEB): return MTBUF
+
+  return None
+
+def disassemble_lib(lib: bytes, compiler) -> list[tuple[str, bytes]]:
+  """Disassemble ELF binary and return list of (instruction_text, machine_code_bytes)."""
+  old_stdout = sys.stdout
+  sys.stdout = io.StringIO()
+  compiler.disassemble(lib)
+  output = sys.stdout.getvalue()
+  sys.stdout = old_stdout
+
+  results = []
+  for line in output.splitlines():
+    if '//' not in line: continue
+    instr = line.split('//')[0].strip()
+    if not instr: continue
+    comment = line.split('//')[1].strip()
+    if ':' not in comment: continue
+    hex_str = comment.split(':')[1].strip().split()[0]
+    try:
+      machine_bytes = bytes.fromhex(hex_str)[::-1]  # big-endian to little-endian
+      results.append((instr, machine_bytes))
+    except ValueError:
+      continue
+  return results
+
+def compile_asm(instr: str, compiler=None) -> bytes:
+  """Compile a single instruction with llvm-mc and return the machine code bytes."""
+  llvm_mc = get_llvm_mc()
+  result = subprocess.run(
+    [llvm_mc, '-triple=amdgcn', '-mcpu=gfx1100', '-mattr=+real-true16,+wavefrontsize32', '-show-encoding'],
+    input=f".text\n{instr}\n", capture_output=True, text=True)
+  if result.returncode != 0: raise RuntimeError(f"llvm-mc failed for '{instr}': {result.stderr.strip()}")
+  # Parse encoding: [0x01,0x39,0x0a,0x7e]
+  for line in result.stdout.split('\n'):
+    if 'encoding:' in line:
+      enc = line.split('encoding:')[1].strip()
+      if enc.startswith('[') and enc.endswith(']'):
+        hex_vals = enc[1:-1].replace('0x', '').replace(',', '').replace(' ', '')
+        return bytes.fromhex(hex_vals)
+  raise RuntimeError(f"no encoding found in llvm-mc output for: {instr}")
+
+def compile_asm_batch(instrs: list[str]) -> list[bytes]:
+  """Compile multiple instructions with a single llvm-mc call."""
+  if not instrs: return []
+  llvm_mc = get_llvm_mc()
+  src = ".text\n" + "\n".join(instrs) + "\n"
+  result = subprocess.run(
+    [llvm_mc, '-triple=amdgcn', '-mcpu=gfx1100', '-mattr=+real-true16,+wavefrontsize32', '-show-encoding'],
+    input=src, capture_output=True, text=True)
+  if result.returncode != 0: raise RuntimeError(f"llvm-mc batch failed: {result.stderr.strip()}")
+  # Parse all encodings in order
+  encodings = []
+  for line in result.stdout.split('\n'):
+    if 'encoding:' in line:
+      enc = line.split('encoding:')[1].strip()
+      if enc.startswith('[') and enc.endswith(']'):
+        hex_vals = enc[1:-1].replace('0x', '').replace(',', '').replace(' ', '')
+        encodings.append(bytes.fromhex(hex_vals))
+  if len(encodings) != len(instrs): raise RuntimeError(f"expected {len(instrs)} encodings, got {len(encodings)}")
+  return encodings
+
+def compile_and_disasm_batch(instrs: list[str], compiler) -> list[str]:
+  """Compile instructions with LLVM and get LLVM's disassembly."""
+  import tempfile, os
+  if not instrs: return []
+  # Build assembly source with all instructions
+  src = ".text\n.globl test\n.p2align 8\n.type test,@function\ntest:\n"
+  src += "\n".join(f"  {instr}" for instr in instrs) + "\n"
+  # Use llvm-mc to assemble to object file
+  with tempfile.NamedTemporaryFile(suffix='.o', delete=False) as f:
+    obj_path = f.name
+  try:
+    result = subprocess.run(
+      [get_llvm_mc(), '-triple=amdgcn', '-mcpu=gfx1100', '-mattr=+real-true16,+wavefrontsize32', '-filetype=obj', '-o', obj_path],
+      input=src, capture_output=True, text=True)
+    if result.returncode != 0: raise RuntimeError(f"llvm-mc failed: {result.stderr.strip()}")
+    # Disassemble with llvm-objdump
+    result = subprocess.run([get_llvm_objdump(), '-d', '--mcpu=gfx1100', obj_path], capture_output=True, text=True)
+    if result.returncode != 0: raise RuntimeError(f"llvm-objdump failed: {result.stderr.strip()}")
+    # Parse disassembly output
+    results: list[str] = []
+    for line in result.stdout.splitlines():
+      if '//' not in line: continue
+      instr = line.split('//')[0].strip()
+      if instr: results.append(instr)
+    return results[:len(instrs)]
+  finally:
+    os.unlink(obj_path)
+
+class TestTinygradKernelRoundtrip(unittest.TestCase):
+  """Test roundtrip on real tinygrad-generated kernels using get_kernels_from_tinygrad pattern."""
+
+  def _test_kernel_roundtrip(self, op_fn):
+    """Generate kernel from op_fn, test:
+    1. decode -> reencode matches original bytes
+    2. asm(disasm()) matches LLVM output
+    3. our disasm() matches LLVM's disassembly string exactly
+    """
+    from extra.assembly.amd.test.test_compare_emulators import get_kernels_from_tinygrad
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+
+    kernels, _, _ = get_kernels_from_tinygrad(op_fn)
+    compiler = HIPCompiler('gfx1100')
+
+    # First pass: decode all instructions and collect info
+    decoded_instrs: list[tuple] = []  # list of (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err)
+    for ki, kernel in enumerate(kernels):
+      offset = 0
+      while offset < len(kernel.code):
+        remaining = kernel.code[offset:]
+        fmt = detect_format(remaining)
+        if fmt is None:
+          decoded_instrs.append((ki, offset, None, None, None, False, "no format"))
+          offset += 4
+          continue
+
+        base_size = fmt._size()
+        if len(remaining) < base_size:
+          break
+
+        try:
+          decoded = fmt.from_bytes(remaining)  # pass all remaining bytes so from_bytes can read literal
+          size = decoded.size()  # actual size including literal
+          orig_bytes = remaining[:size]
+          reencoded = decoded.to_bytes()
+          our_disasm = decoded.disasm()
+          decode_ok = reencoded == orig_bytes
+          decode_err: str | None = None if decode_ok else f"orig={orig_bytes.hex()} reenc={reencoded.hex()}"
+          decoded_instrs.append((ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err))
+        except Exception as e:
+          decoded_instrs.append((ki, offset, remaining[:base_size], None, None, False, str(e)))
+          size = base_size
+
+        offset += size
+
+    # Collect disasm strings for batched LLVM calls - skip unknown opcodes (op_X) that LLVM can't compile
+    asm_test_instrs: list[tuple[int, str]] = []  # (idx, our_disasm) for asm test
+    disasm_test_instrs: list[tuple[int, str]] = []  # (idx, our_disasm) for disasm comparison test
+
+    for idx, (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err) in enumerate(decoded_instrs):
+      if our_disasm is None: continue
+      # Skip unknown opcodes and malformed instructions for both tests
+      if our_disasm.startswith('op_') or re.search(r', \d+, \d+, \d+,', our_disasm): continue
+      asm_test_instrs.append((idx, our_disasm))
+      disasm_test_instrs.append((idx, our_disasm))
+
+    # Batch compile for asm test
+    asm_llvm_results = compile_asm_batch([d for _, d in asm_test_instrs])
+    asm_llvm_map = {idx: result for (idx, _), result in zip(asm_test_instrs, asm_llvm_results)}
+
+    # Batch compile+disasm for disasm comparison test
+    disasm_llvm_results = compile_and_disasm_batch([d for _, d in disasm_test_instrs], compiler)
+    disasm_llvm_map = {idx: result for (idx, _), result in zip(disasm_test_instrs, disasm_llvm_results)}
+
+    # Now evaluate results
+    decode_passed, decode_failed, decode_skipped = 0, 0, 0
+    asm_passed, asm_failed, asm_skipped = 0, 0, 0
+    disasm_passed, disasm_failed, disasm_skipped = 0, 0, 0
+    decode_failures: list[str] = []
+    asm_failures: list[str] = []
+    disasm_failures: list[str] = []
+
+    for idx, (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err) in enumerate(decoded_instrs):
+      # Decode test
+      if decode_ok:
+        decode_passed += 1
+      elif decode_err == "no format":
+        decode_skipped += 1
+      else:
+        decode_failed += 1
+        decode_failures.append(f"K{ki}@{offset}: {our_disasm}: {decode_err}")
+
+      # Asm test
+      if our_disasm is None:
+        asm_skipped += 1
+      elif idx in asm_llvm_map:
+        llvm_bytes = asm_llvm_map[idx]
+        try:
+          our_bytes = asm(our_disasm).to_bytes()
+          if our_bytes[:len(llvm_bytes)] == llvm_bytes:
+            asm_passed += 1
+          else:
+            asm_failed += 1
+            asm_failures.append(f"K{ki}@{offset}: '{our_disasm}': ours={our_bytes[:len(llvm_bytes)].hex()} llvm={llvm_bytes.hex()}")
+        except Exception:
+          asm_skipped += 1
+      else:
+        asm_skipped += 1
+
+      # Disasm comparison test
+      if our_disasm is None:
+        disasm_skipped += 1
+      elif idx in disasm_llvm_map:
+        llvm_disasm = disasm_llvm_map[idx]
+        if our_disasm == llvm_disasm:
+          disasm_passed += 1
+        else:
+          disasm_failed += 1
+          disasm_failures.append(f"K{ki}@{offset}: ours='{our_disasm}' llvm='{llvm_disasm}'")
+      else:
+        disasm_skipped += 1
+
+    print(f"decode roundtrip: {decode_passed} passed, {decode_failed} failed, {decode_skipped} skipped")
+    print(f"asm vs llvm: {asm_passed} passed, {asm_failed} failed, {asm_skipped} skipped")
+    print(f"disasm vs llvm: {disasm_passed} passed, {disasm_failed} failed, {disasm_skipped} skipped")
+    self.assertEqual(decode_failed, 0, f"Decode failures:\n" + "\n".join(decode_failures[:20]))
+    self.assertEqual(asm_failed, 0, f"Asm failures:\n" + "\n".join(asm_failures[:20]))
+    # Note: disasm string comparison is informational only - formatting differences between LLVM versions are expected
+
+  # Basic unary ops
+  def test_neg(self): self._test_kernel_roundtrip(lambda T: -T([1.0, -2.0, 3.0, -4.0]))
+  def test_relu(self): self._test_kernel_roundtrip(lambda T: T([-1.0, 0.0, 1.0, 2.0]).relu())
+  def test_exp(self): self._test_kernel_roundtrip(lambda T: T([0.0, 1.0, 2.0]).exp())
+  def test_log(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0, 3.0]).log())
+  def test_sin(self): self._test_kernel_roundtrip(lambda T: T([0.0, 1.0, 2.0]).sin())
+  def test_sqrt(self): self._test_kernel_roundtrip(lambda T: T([1.0, 4.0, 9.0]).sqrt())
+  def test_recip(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0, 4.0]).reciprocal())
+
+  # Binary ops
+  def test_add(self): self._test_kernel_roundtrip(lambda T: T([1.0, 2.0]) + T([3.0, 4.0]))
+  def test_sub(self): self._test_kernel_roundtrip(lambda T: T([5.0, 6.0]) - T([1.0, 2.0]))
+  def test_mul(self): self._test_kernel_roundtrip(lambda T: T([2.0, 3.0]) * T([4.0, 5.0]))
+  def test_div(self): self._test_kernel_roundtrip(lambda T: T([10.0, 20.0]) / T([2.0, 4.0]))
+  def test_max_binary(self): self._test_kernel_roundtrip(lambda T: T([1.0, 5.0]).maximum(T([3.0, 2.0])))
+
+  # Reductions
+  def test_sum_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(64).sum())
+  def test_max_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(64).max())
+  def test_mean_reduce(self): self._test_kernel_roundtrip(lambda T: T.empty(32).mean())
+
+  # Matmul
+  def test_gemm_4x4(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 4) @ T.empty(4, 4))
+  def test_gemv(self): self._test_kernel_roundtrip(lambda T: T.empty(1, 16) @ T.empty(16, 16))
+
+  # Complex ops
+  def test_softmax(self): self._test_kernel_roundtrip(lambda T: T.empty(16).softmax())
+  def test_layernorm(self): self._test_kernel_roundtrip(lambda T: T.empty(8, 8).layernorm())
+
+  # Memory patterns
+  def test_contiguous(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 4).permute(1, 0).contiguous())
+  def test_reshape(self): self._test_kernel_roundtrip(lambda T: (T.empty(16) + 1).reshape(4, 4).contiguous())
+  def test_expand(self): self._test_kernel_roundtrip(lambda T: T.empty(4, 1).expand(4, 4).contiguous())
+
+  # Cast ops
+  def test_cast_int(self): self._test_kernel_roundtrip(lambda T: T.empty(16).int().float())
+  def test_cast_half(self): self._test_kernel_roundtrip(lambda T: T.empty(16).half().float())
+
+  # Comparison ops
+  def test_cmp_lt(self): self._test_kernel_roundtrip(lambda T: (T.empty(64) < T.empty(64)).where(T.empty(64), T.empty(64)))
+  def test_where(self): self._test_kernel_roundtrip(lambda T: (T.empty(64) > 0).where(T.empty(64), T.empty(64)))
+
+  # Fused ops
+  def test_fma(self): self._test_kernel_roundtrip(lambda T: (T([1.0, 2.0]) * T([3.0, 4.0]) + T([5.0, 6.0])))
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/rdna3/generate.py

@@ -1,136 +0,0 @@
-import os, sys, struct
-sys.path.append(os.getcwd())
-# PROFILE=1 to use
-#os.environ["PROFILE"] = "1"
-os.environ["SQTT"] = "1"
-os.environ["SQTT_ITRACE_SE_MASK"] = "1"
-os.environ["SQTT_LIMIT_SE"] = "1"
-import xml.etree.ElementTree as ET
-
-from tinygrad import nn, Tensor, Device
-from tinygrad.helpers import get_single_element
-from tinygrad.runtime.support.elf import elf_loader
-from tinygrad.runtime.ops_amd import ProfileSQTTEvent
-from extra.sqtt.attempt_sqtt_parse import parse_sqtt_print_packets
-
-def disassemble(text, root:ET.Element):
-  i = 0
-  while i < len(text):
-    ins = struct.unpack("I", text[i:i+4])[0]
-
-    # 1. Get the encoding
-    did_match = False
-    for enc_el in root.findall("./ISA/Encodings/Encoding"):
-      mask = enc_el.findtext("EncodingIdentifierMask")
-      assert len(mask)%32 == 0
-      bit_mask = int(mask, 2)
-      iden = [int(x.text, 2) for x in enc_el.find("EncodingIdentifiers").findall("EncodingIdentifier")]
-      for ide in iden:
-        if ins&bit_mask == ide:
-          did_match = True
-          break
-      if did_match: break
-    if not did_match: raise RuntimeError(f"unknown instruction {ins:08X}")
-    if len(mask) >= 64: ins = (struct.unpack("I", text[i+4:i+8])[0]<<32) | ins
-    if len(mask) >= 96: ins = (struct.unpack("I", text[i+8:i+12])[0]<<64) | ins
-    encoding_name = enc_el.findtext("EncodingName")
-
-    #print(ET.tostring(enc_el).decode())
-
-    # 2. Parse the Fields for this Encoding
-    field_data = {}
-    for field in enc_el.findall("MicrocodeFormat/BitMap/Field"):
-      # Fields can be split into multiple ranges (RangeCount > 1)
-      ranges = sorted(field.findall("BitLayout/Range"), key=lambda x: int(x.attrib.get('Order')))
-      val = 0
-      current_shift = 0
-      for rng in ranges:
-        width = int(rng.find("BitCount").text)
-        chunk = (ins >> int(rng.find("BitOffset").text)) & ((1 << width) - 1)
-        val |= (chunk << current_shift)
-        current_shift += width
-      field_data[field.find("FieldName").text] = val
-    # this is already used
-    del field_data["ENCODING"]
-
-    # 3. Extract the instruction
-    did_match = False
-    for ins_el in root.findall("./ISA/Instructions/Instruction"):
-      ins_name = ins_el.findtext("InstructionName")
-      for ins_enc in ins_el.findall("InstructionEncodings/InstructionEncoding"):
-        if ins_enc.findtext("EncodingName") == encoding_name:
-          opcode = int(ins_enc.findtext("Opcode"))
-          if "OP" in field_data and opcode == field_data["OP"]:
-            did_match = True
-            del field_data["OP"]
-            break
-        if did_match: break
-      if did_match: break
-
-    #print(ET.tostring(ins_enc).decode())
-    #print()
-    #print(field_data)
-    if not did_match:
-      print(f"{i:4X} : {ins:16x} -- {encoding_name}")
-    elif did_match:
-      params = []
-      #print(ET.tostring(ins_el).decode())
-
-      # 4. Extract the opcodes
-      for op_ins in ins_enc.findall("Operands/Operand"):
-        op_type = op_ins.findtext("OperandType")
-        op_size = op_ins.findtext("OperandSize")
-        op_fmt = op_ins.findtext("DataFormatName")
-        op_field_name = op_ins.findtext("FieldName")
-        if op_field_name is None: continue
-        assert op_field_name in field_data
-        # loop through operands for compare
-        for op_el in root.findall("./ISA/OperandTypes/OperandType"):
-          test_op_type = op_el.findtext("OperandTypeName")
-          val_dict = {}
-          for op_val in op_el.findall("OperandPredefinedValues/PredefinedValue"):
-            val_dict[int(op_val.findtext("Value"))] = op_val.findtext("Name")
-          if op_type == test_op_type:
-            if field_data[op_field_name] in val_dict:
-              print(op_type, op_size, op_fmt)
-              params.append(val_dict[field_data[op_field_name]])
-            else:
-              params.append(f"{op_type}({field_data[op_field_name]})")
-            del field_data[op_field_name]
-            #print(op_type, op_size, op_fmt, op_el, op_field_name,
-            #      field_data[op_field_name],
-            #      val_dict.get(field_data[op_field_name], "<UNK>"))
-            #print(ET.tostring(op_el).decode())
-
-      print(f"{i:4X} : {ins:16x} -- {ins_name.lower()} {', '.join(params)}", field_data)
-
-    # advance
-    i += len(mask) // 8
-
-  #print(ET.tostring(root).decode())
-
-if __name__ == "__main__":
-  # human readable manual at https://docs.amd.com/v/u/en-US/rdna35_instruction_set_architecture
-  fns = nn.state.zip_extract(Tensor.from_url("https://gpuopen.com/download/machine-readable-isa/latest/"))
-  xml_str = fns['amdgpu_isa_rdna3_5.xml'].to("CPU").data()
-  with open("/tmp/rdna35.xml", "wb") as f: f.write(bytes(xml_str))
-  root = ET.fromstring(xml_str)
-
-  a = Tensor.empty(16)+1
-  for ei in a.schedule():
-    ei.lower()
-    # get text
-    _, hdr, _ = elf_loader(ei.prg.lib)
-    text = get_single_element([x for x in hdr if x.name==".text"]).content
-
-    # llvm disassembler
-    Device["AMD"].compiler.disassemble(ei.prg.lib)
-
-    # run program
-    ei.run()
-
-  sqtt_events = [e for e in Device["AMD"].profile_events if isinstance(e, ProfileSQTTEvent)]
-  for e in sqtt_events[0:1]: # only the first SE
-    parse_sqtt_print_packets(e.blob)
-
-  disassemble(text[:0x40], root)

extra/assembly/rdna3/parse.py

@@ -1,15 +0,0 @@
-from tinygrad import Tensor, nn
-import xml.etree.ElementTree as ET
-
-if __name__ == "__main__":
-  # human readable manual at https://docs.amd.com/v/u/en-US/rdna35_instruction_set_architecture
-  fns = nn.state.zip_extract(Tensor.from_url("https://gpuopen.com/download/machine-readable-isa/latest/"))
-  xml_str = fns['amdgpu_isa_rdna3_5.xml'].to("CPU").data()
-  root = ET.fromstring(xml_str)
-
-  for op_el in root.findall("./ISA/OperandTypes/OperandType"):
-    op_name = op_el.findtext("OperandTypeName")
-    val_dict = {}
-    for op_val in op_el.findall("OperandPredefinedValues/PredefinedValue"):
-      val_dict[int(op_val.findtext("Value"))] = op_val.findtext("Name")
-    print(op_name, val_dict)

extra/assembly/rocm/.gitignore

@@ -1,4 +0,0 @@
-*.deb
-build
-src
-sniffer/sniff.so

extra/assembly/rocm/NOTES

@@ -1,20 +0,0 @@
-Built ROCT-Thunk-Interface (hsakmt)
-  hsakmt-roct-dev_5.4.4.99999-local_amd64.deb
-  note: installs to /opt/rocm
-Built ROCm-Device-Libs
-  Works with ROCM_PATH=/home/tiny/build/ROCm-Device-Libs/build/dist
-  rocm-device-libs_1.0.0.99999-local_amd64.deb
-Built ROCm-CompilerSupport (amd_comgr)
-  no deb, sudo make install to /usr/local
-Built ROCR-Runtime
-  hsa-rocr_1.8.0-local_amd64.deb
-  hsa-rocr-dev_1.8.0-local_amd64.deb
-Built ROCm-OpenCL-Runtime
-  rocm-ocl-icd_2.0.0-local_amd64.deb
-  ISSUE: these depend on "comgr"
-  rocm-opencl_2.0.0-local_amd64.deb
-  rocm-opencl-dev_2.0.0-local_amd64.deb
-  Did sudo make install
-
-
-

extra/assembly/rocm/kernel_crashes/dump

@@ -1,41 +0,0 @@
-# run two "rocm-bandwidth-test" in a loop
-# amdgpu-6.0.5-1581431.20.04
-# fixed in kernel 6.2.14
-
-[   72.153646] RIP: 0010:pm_send_runlist+0x4a/0x630 [amdgpu]
-[   72.153815] Code: 30 65 48 8b 04 25 28 00 00 00 48 89 45 d0 31 c0 80 fb 01 0f 87 aa 9d 49 00 83 e3 01 0f 85 1c 05 00 00 49 8b 3f b8 01 00 00 00 <48> 8b 97 30 01 00 00 44 8b b7 6c 01 00 00 8b 9f 70 01 00 00 8b 8a
-[   72.153900] RSP: 0018:ffffb48445c03c30 EFLAGS: 00010246
-[   72.153928] RAX: 0000000000000001 RBX: 0000000000000000 RCX: 0000000000000000
-[   72.153962] RDX: 000000000000007b RSI: ffff9395e1562558 RDI: 0000000000000000
-[   72.153996] RBP: ffffb48445c03cb8 R08: 0000000000000000 R09: 0000000000000001
-[   72.154030] R10: ffff9395c900d840 R11: 0000000000000000 R12: 0000000000000000
-[   72.154065] R13: ffff9395c9e00400 R14: 0000000000000001 R15: ffff9395e15624e0
-[   72.154099] FS:  00007f345c6463c0(0000) GS:ffff93a4aee80000(0000) knlGS:0000000000000000
-[   72.154137] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
-[   72.154165] CR2: 0000000000000130 CR3: 0000000112840000 CR4: 0000000000750ee0
-[   72.154201] PKRU: 55555554
-[   72.154215] Call Trace:
-[   72.154230]  <TASK>
-[   72.154244]  map_queues_cpsch+0x75/0xc0 [amdgpu]
-[   72.154365]  debug_map_and_unlock+0x51/0x90 [amdgpu]
-[   72.154480]  debug_refresh_runlist+0x1f/0x30 [amdgpu]
-[   72.154591]  kfd_dbg_runtime_disable+0x13c/0x240 [amdgpu]
-[   72.154705]  kfd_ioctl_dbg_set_debug_trap+0x69d/0x8b0 [amdgpu]
-[   72.154820]  kfd_ioctl+0x24a/0x5b0 [amdgpu]
-[   72.154925]  ? kfd_ioctl_create_queue+0x770/0x770 [amdgpu]
-[   72.155035]  ? syscall_exit_to_user_mode+0x27/0x50
-[   72.155061]  ? exit_to_user_mode_prepare+0x3d/0x1c0
-[   72.155088]  __x64_sys_ioctl+0x95/0xd0
-[   72.155109]  do_syscall_64+0x5c/0xc0
-[   72.155128]  ? syscall_exit_to_user_mode+0x27/0x50
-[   72.155151]  ? do_syscall_64+0x69/0xc0
-[   72.155172]  entry_SYSCALL_64_after_hwframe+0x61/0xcb
-[   72.155198] RIP: 0033:0x7f345c7f63ab
-[   72.155218] Code: 0f 1e fa 48 8b 05 e5 7a 0d 00 64 c7 00 26 00 00 00 48 c7 c0 ff ff ff ff c3 66 0f 1f 44 00 00 f3 0f 1e fa b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d b5 7a 0d 00 f7 d8 64 89 01 48
-[   72.155301] RSP: 002b:00007ffc97cc89f8 EFLAGS: 00000246 ORIG_RAX: 0000000000000010
-[   72.155339] RAX: ffffffffffffffda RBX: 00007ffc97cc8a30 RCX: 00007f345c7f63ab
-[   72.155375] RDX: 00007ffc97cc8a30 RSI: 00000000c0284b82 RDI: 0000000000000003
-[   72.155411] RBP: 00000000c0284b82 R08: 0000000000000000 R09: 0000000000000000
-[   72.155447] R10: 00007f345cd4ddb0 R11: 0000000000000246 R12: 00007ffc97cc8a30
-[   72.155481] R13: 0000000000000003 R14: 00007ffc97cc8d20 R15: 0000000000000000
-[   72.155517]  </TASK>

extra/assembly/rocm/kernel_crashes/dump2

@@ -1,41 +0,0 @@
-# run two tinygrad matrix example in a loop
-# amdgpu-6.0.5-1581431.20.04
-# NOT fixed in kernel 6.2.14
-
-[  553.016624] gmc_v11_0_process_interrupt: 30 callbacks suppressed
-[  553.016631] amdgpu 0000:0b:00.0: amdgpu: [gfxhub] page fault (src_id:0 ring:24 vmid:9 pasid:32770, for process python3 pid 10001 thread python3 pid 10001)
-[  553.016790] amdgpu 0000:0b:00.0: amdgpu:   in page starting at address 0x00007f0000000000 from client 10
-[  553.016892] amdgpu 0000:0b:00.0: amdgpu: GCVM_L2_PROTECTION_FAULT_STATUS:0x00901A30
-[  553.016974] amdgpu 0000:0b:00.0: amdgpu:      Faulty UTCL2 client ID: SDMA0 (0xd)
-[  553.017051] amdgpu 0000:0b:00.0: amdgpu:      MORE_FAULTS: 0x0
-[  553.017111] amdgpu 0000:0b:00.0: amdgpu:      WALKER_ERROR: 0x0
-[  553.017173] amdgpu 0000:0b:00.0: amdgpu:      PERMISSION_FAULTS: 0x3
-[  553.017238] amdgpu 0000:0b:00.0: amdgpu:      MAPPING_ERROR: 0x0
-[  553.017300] amdgpu 0000:0b:00.0: amdgpu:      RW: 0x0
-[  553.123921] [drm:mes_v11_0_submit_pkt_and_poll_completion.constprop.0 [amdgpu]] *ERROR* MES failed to response msg=2
-[  553.124153] amdgpu: failed to add hardware queue to MES, doorbell=0x1a16
-[  553.124195] amdgpu: MES might be in unrecoverable state, issue a GPU reset
-[  553.124237] amdgpu: Failed to restore queue 2
-[  553.124266] amdgpu: Failed to restore process queues
-[  553.124270] amdgpu: Failed to evict queue 3
-[  553.124297] amdgpu: amdgpu_amdkfd_restore_userptr_worker: Failed to resume KFD
-
-# alternative crash in kernel 6.2.14
-
-[  151.097948] gmc_v11_0_process_interrupt: 30 callbacks suppressed
-[  151.097953] amdgpu 0000:0b:00.0: amdgpu: [gfxhub] page fault (src_id:0 ring:24 vmid:8 pasid:32771, for process python3 pid 7525 thread python3 pid 7525)
-[  151.097993] amdgpu 0000:0b:00.0: amdgpu:   in page starting at address 0x00007f0000000000 from client 10
-[  151.098008] amdgpu 0000:0b:00.0: amdgpu: GCVM_L2_PROTECTION_FAULT_STATUS:0x00801A30
-[  151.098020] amdgpu 0000:0b:00.0: amdgpu:      Faulty UTCL2 client ID: SDMA0 (0xd)
-[  151.098032] amdgpu 0000:0b:00.0: amdgpu:      MORE_FAULTS: 0x0
-[  151.098042] amdgpu 0000:0b:00.0: amdgpu:      WALKER_ERROR: 0x0
-[  151.098052] amdgpu 0000:0b:00.0: amdgpu:      PERMISSION_FAULTS: 0x3
-[  151.098062] amdgpu 0000:0b:00.0: amdgpu:      MAPPING_ERROR: 0x0
-[  151.098071] amdgpu 0000:0b:00.0: amdgpu:      RW: 0x0
-[  151.209517] [drm:mes_v11_0_submit_pkt_and_poll_completion.constprop.0 [amdgpu]] *ERROR* MES failed to response msg=2
-[  151.209724] amdgpu: failed to add hardware queue to MES, doorbell=0x1002
-[  151.209734] amdgpu: MES might be in unrecoverable state, issue a GPU reset
-[  151.209743] amdgpu: Failed to restore queue 1
-[  151.209751] amdgpu: Failed to restore process queues
-[  151.209759] amdgpu: amdgpu_amdkfd_restore_userptr_worker: Failed to resume KFD
-[  151.209858] amdgpu 0000:0b:00.0: amdgpu: GPU reset begin!

extra/assembly/rocm/kernel_crashes/dump3

@@ -1,20 +0,0 @@
-# two tinygrad + two bandwidth test
-# RDNA2, driver 6.0.5
-# recovered from this!
-
-[  136.971209] gmc_v10_0_process_interrupt: 39 callbacks suppressed
-[  136.971218] amdgpu 0000:0b:00.0: amdgpu: [gfxhub] page fault (src_id:0 ring:24 vmid:11 pasid:32773, for process rocm-bandwidth- pid 20281 thread rocm-bandwidth- pid 20281)
-[  136.971228] amdgpu 0000:0b:00.0: amdgpu:   in page starting at address 0x00007f5c2b800000 from client 0x1b (UTCL2)
-[  136.971232] amdgpu 0000:0b:00.0: amdgpu: GCVM_L2_PROTECTION_FAULT_STATUS:0x00B01A31
-[  136.971233] amdgpu 0000:0b:00.0: amdgpu:      Faulty UTCL2 client ID: SDMA0 (0xd)
-[  136.971235] amdgpu 0000:0b:00.0: amdgpu:      MORE_FAULTS: 0x1
-[  136.971236] amdgpu 0000:0b:00.0: amdgpu:      WALKER_ERROR: 0x0
-[  136.971236] amdgpu 0000:0b:00.0: amdgpu:      PERMISSION_FAULTS: 0x3
-[  136.971237] amdgpu 0000:0b:00.0: amdgpu:      MAPPING_ERROR: 0x0
-[  136.971238] amdgpu 0000:0b:00.0: amdgpu:      RW: 0x0
-...
-[  136.993979] amdgpu 0000:0b:00.0: amdgpu: IH ring buffer overflow (0x000BE5A0, 0x0003C480, 0x0003E5C0)
-[  138.209072] amdgpu 0000:0b:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x001a address=0x7c00004000 flags=0x0000]
-[  138.209078] amdgpu 0000:0b:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x001a address=0x7c00004d80 flags=0x0000]
-[  138.209081] amdgpu 0000:0b:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x001a address=0x7c00005000 flags=0x0000]
-[  138.209084] amdgpu 0000:0b:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x001a address=0x7c00005d80 flags=0x0000]

extra/assembly/rocm/kernel_crashes/dump4

@@ -1,33 +0,0 @@
-# ROCK-Kernel-Driver 0b579de9622f5c93021dcb7927d13926313740a2
-# non fatal "crash"
-
-[  127.418045] ------------[ cut here ]------------
-[  127.418046] User pages unexpectedly invalid
-[  127.418056] WARNING: CPU: 16 PID: 260 at drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c:3000 amdgpu_amdkfd_restore_userptr_worker+0x4d9/0x500 [amdgpu]
-[  127.418235] Modules linked in: rfcomm cmac algif_hash algif_skcipher af_alg bnep nls_iso8859_1 iwlmvm mac80211 intel_rapl_msr intel_rapl_common edac_mce_amd snd_hda_codec_realtek snd_hda_codec_generic snd_hda_codec_hdmi kvm_amd binfmt_misc snd_hda_intel snd_intel_dspcfg kvm libarc4 snd_intel_sdw_acpi snd_hda_codec btusb iwlwifi btrtl snd_hda_core btbcm btintel irqbypass btmtk snd_hwdep crct10dif_pclmul snd_pcm polyval_clmulni bluetooth snd_seq_midi snd_seq_midi_event snd_rawmidi snd_seq polyval_generic cfg80211 ghash_clmulni_intel eeepc_wmi snd_seq_device snd_timer aesni_intel asus_wmi ecdh_generic snd platform_profile crypto_simd ledtrig_audio cryptd ecc ccp soundcore sparse_keymap rapl k10temp wmi_bmof mac_hid sch_fq_codel msr parport_pc ppdev lp parport ramoops pstore_blk efi_pstore reed_solomon pstore_zone ip_tables x_tables autofs4 amdgpu hid_generic usbhid hid i2c_algo_bit drm_ttm_helper ttm video iommu_v2 drm_buddy gpu_sched drm_display_helper drm_kms_helper syscopyarea
-[  127.418276]  sysfillrect sysimgblt fb_sys_fops drm nvme nvme_core cec r8169 ahci crc32_pclmul rc_core i2c_piix4 xhci_pci libahci nvme_common xhci_pci_renesas realtek wmi
-[  127.418284] CPU: 16 PID: 260 Comm: kworker/16:1 Tainted: G        W          6.0.0 #4
-[  127.418286] Hardware name: System manufacturer System Product Name/TUF GAMING X570-PLUS (WI-FI), BIOS 3603 03/20/2021
-[  127.418287] Workqueue: events amdgpu_amdkfd_restore_userptr_worker [amdgpu]
-[  127.418455] RIP: 0010:amdgpu_amdkfd_restore_userptr_worker+0x4d9/0x500 [amdgpu]
-[  127.418601] Code: ff e8 2b 8a 96 d1 e9 66 fe ff ff 48 c7 c7 40 4f f5 c0 e8 56 7b 8a d1 0f 0b e9 2e ff ff ff 48 c7 c7 d8 d0 ed c0 e8 43 7b 8a d1 <0f> 0b e9 0a fe ff ff 4c 89 ef e8 f8 89 96 d1 e9 cb fd ff ff e8 ce
-[  127.418603] RSP: 0018:ffffb36740a83dc8 EFLAGS: 00010282
-[  127.418604] RAX: 0000000000000000 RBX: ffff9d159ee9df30 RCX: 0000000000000027
-[  127.418605] RDX: 0000000000000027 RSI: ffffb36740a83c88 RDI: ffff9d242a220568
-[  127.418606] RBP: ffffb36740a83e58 R08: ffff9d242a220560 R09: 0000000000000001
-[  127.418607] R10: 0000000000000001 R11: 0000000000000020 R12: ffff9d159ee9df98
-[  127.418607] R13: ffff9d159ee9df70 R14: ffff9d159ee9dee0 R15: ffff9d159ee9dee0
-[  127.418608] FS:  0000000000000000(0000) GS:ffff9d242a200000(0000) knlGS:0000000000000000
-[  127.418609] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
-[  127.418610] CR2: 00007fd5d4715000 CR3: 0000000120ffe000 CR4: 0000000000750ee0
-[  127.418611] PKRU: 55555554
-[  127.418611] Call Trace:
-[  127.418612]  <TASK>
-[  127.418613]  process_one_work+0x21f/0x3f0
-[  127.418615]  worker_thread+0x4a/0x3c0
-[  127.418617]  ? process_one_work+0x3f0/0x3f0
-[  127.418618]  kthread+0xf0/0x120
-[  127.418619]  ? kthread_complete_and_exit+0x20/0x20
-[  127.418620]  ret_from_fork+0x22/0x30
-[  127.418622]  </TASK>
-[  127.418623] ---[ end trace 0000000000000000 ]---

extra/assembly/rocm/rdna3/asm.py

@@ -1,80 +0,0 @@
-import numpy as np
-import pathlib
-from hexdump import hexdump
-from tinygrad.helpers import colored
-from extra.helpers import enable_early_exec
-early_exec = enable_early_exec()
-
-from tinygrad.runtime.ops_cl import CLProgram, CLBuffer, ROCM_LLVM_PATH
-
-ENABLE_NON_ASM = False
-
-WMMA = True
-DUAL_ALU = True
-F32 = True
-
-if ENABLE_NON_ASM:
-  buf = CLBuffer.fromCPU(np.zeros(10, np.float32))
-  prg_empty = CLProgram("code", "__kernel void code(__global float *a) { a[0] = 1; }")
-  asm_real = prg_empty.binary()
-  with open("/tmp/cc.elf", "wb") as f:
-    f.write(asm_real)
-  prg_empty([1], [1], buf, wait=True)
-  print(buf.toCPU())
-
-print(colored("creating CLBuffer", "green"))
-buf = CLBuffer.fromCPU(np.zeros(10, np.float32))
-code = open(pathlib.Path(__file__).parent / "prog.s", "r").read()
-
-gen = []
-FLOPS = 0
-MAX_REG = 251
-for j in range(1):
-  if WMMA:
-    KY, KX = 4, 4
-    for y in range(KY):
-      for x in range(KX):
-        c = (y*KX+x)*8
-        a = (KY*KX*8) + y*8
-        b = (KY*KX*8) + (KY*8) + x*8
-        gen.append(f"v_wmma_f32_16x16x16_f16 v[{c}:{c+7}], v[{a}:{a+7}], v[{b}:{b+7}], v[{c}:{c+7}]")
-        FLOPS += 16*8*2
-  else:
-    for i in range(0, MAX_REG, 6):
-      if DUAL_ALU:
-        if F32:
-          gen.append(f"v_dual_fmac_f32 v{i+0}, v{i+1}, v{i+2} :: v_dual_fmac_f32 v{i+3}, v{i+4}, v{i+5}")
-          FLOPS += 4
-        else:
-          gen.append(f"v_dual_dot2acc_f32_f16 v{i+0}, v{i+1}, v{i+2} :: v_dual_dot2acc_f32_f16 v{i+3}, v{i+4}, v{i+5}")
-          FLOPS += 8
-      else:
-        assert F32
-        gen.append(f"v_fmac_f32 v{i+0}, v{i+1}, v{i+2}")
-        gen.append(f"v_fmac_f32 v{i+3}, v{i+4}, v{i+5}")
-code = code.replace("// FLOPS", '\n'.join(gen))
-print(code)
-
-
-# fix: COMGR failed to get code object ISA name. set triple to 'amdgcn-amd-amdhsa'
-
-object = early_exec(([ROCM_LLVM_PATH / "llvm-mc", '--arch=amdgcn', '--mcpu=gfx1100', '--triple=amdgcn-amd-amdhsa', '--filetype=obj', '-'], code.encode("utf-8")))
-asm = early_exec(([ROCM_LLVM_PATH / "ld.lld", "/dev/stdin", "-o", "/dev/stdout", "--pie"], object))
-
-with open("/tmp/cc2.o", "wb") as f:
-  f.write(object)
-with open("/tmp/cc2.elf", "wb") as f:
-  f.write(asm)
-
-print(colored("creating CLProgram", "green"))
-prg = CLProgram("code", asm)
-
-print(colored("running program", "green"))
-G = 512
-FLOPS *= 100000*G*G  # loop * global_size
-for i in range(3):
-  tm = prg(buf, global_size=[G//256, G, 1], local_size=[256, 1, 1], wait=True)
-  print(f"ran in {tm*1e3:.2f} ms, {FLOPS/(tm*1e9):.2f} GFLOPS")
-
-print(colored("transferring buffer", "green"))
-print(buf.toCPU())

extra/assembly/rocm/rdna3/prog.s

@@ -1,80 +0,0 @@
-.global _start
-_start:
-.rodata
-.align 0x10
-.global code.kd
-.type code.kd,STT_OBJECT
-# amd_kernel_code_t (must be at 0x440 for kernel_code_entry_byte_offset to be right)
-code.kd:
-# amd_kernel_..., amd_machine_...
-.long 0,0,0,0
-# kernel_code_entry_byte_offset, kernel_code_prefetch_byte_offset
-.long 0x00000bc0,0x00000000,0x00000000,0x00000000
-# kernel_code_prefetch_byte_size, max_scratch_backing_memory_byte_size
-.long 0,0,0,0
-# compute_pgm_rsrc1, compute_pgm_rsrc2, kernel_code_properties, workitem_private_segment_byte_size
-.long 0x60af0000,0x0000009e,0x00000408,0x00000000
-# compute_pgm_rsrc1 |= AMD_COMPUTE_PGM_RSRC_ONE_FLOAT_DENORM_MODE_32 | AMD_COMPUTE_PGM_RSRC_ONE_FLOAT_DENORM_MODE_16_64
-# compute_pgm_rsrc1 |= AMD_COMPUTE_PGM_RSRC_ONE_ENABLE_DX10_CLAMP | AMD_COMPUTE_PGM_RSRC_ONE_ENABLE_IEEE_MODE
-# compute_pgm_rsrc2 |= AMD_COMPUTE_PGM_RSRC_TWO_USER_SGPR_COUNT = 0xF
-# compute_pgm_rsrc2 |= AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_X
-# kernel_code_properties |= AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_KERNARG_SEGMENT_PTR = 1
-# kernel_code_properties |= AMD_KERNEL_CODE_PROPERTIES_RESERVED1 = 1
-.text
-.global code
-.type code,STT_FUNC
-code:
-# https://llvm.org/docs/AMDGPUUsage.html#initial-kernel-execution-state
-# s[0:1] contains the kernarg_address
-# TODO: can we use s[2:3] if this was really a wave since we only alloced 2 SGPRs?
-s_load_b64 s[2:3], s[0:1], null
-
-s_mov_b32 s8, 0
-loop:
-s_addk_i32 s8, 1
-s_cmp_eq_u32 s8, 100000
-// FLOPS
-s_cbranch_scc0 loop
-
-# wait for the s_load_b64
-s_waitcnt lgkmcnt(0)
-
-v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 2.0
-global_store_b32 v0, v1, s[2:3]
-
-# Deallocate all VGPRs for this wave. Use only when next instruction is S_ENDPGM.
-s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-s_endpgm
-s_code_end
-
-.amdgpu_metadata
-amdhsa.kernels:
-  - .args:
-      - .address_space:  global
-        .name:           a
-        .offset:         0
-        .size:           8
-        .type_name:      'float*'
-        .value_kind:     global_buffer
-    .group_segment_fixed_size: 0
-    .kernarg_segment_align: 8
-    .kernarg_segment_size: 8
-    .language:       OpenCL C
-    .language_version:
-      - 1
-      - 2
-    .max_flat_workgroup_size: 256
-    .name:           code
-    .private_segment_fixed_size: 0
-    .sgpr_count:     2
-    .sgpr_spill_count: 0
-    .symbol:         code.kd
-    .uses_dynamic_stack: false
-    .vgpr_count:     256
-    .vgpr_spill_count: 0
-    .wavefront_size: 32
-amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
-amdhsa.version:
-  - 1
-  - 2
-.end_amdgpu_metadata

extra/assembly/rocm/rocm_clone.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-mkdir -p src
-cd src
-git clone https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface.git -b rocm-5.5.0
-git clone https://github.com/RadeonOpenCompute/ROCm-Device-Libs.git -b rocm-5.5.0
-git clone https://github.com/RadeonOpenCompute/llvm-project.git -b rocm-5.5.0 --depth 1
-git clone https://github.com/RadeonOpenCompute/ROCR-Runtime.git -b rocm-5.5.0
-git clone https://github.com/ROCm-Developer-Tools/ROCclr.git -b rocm-5.5.0
-git clone https://github.com/RadeonOpenCompute/ROCm-CompilerSupport.git -b rocm-5.5.0
-git clone https://github.com/RadeonOpenCompute/ROCm-OpenCL-Runtime.git -b rocm-5.5.0
-cd ../

extra/assembly/rocm/rocm_from_scratch.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-mkdir -p build/debs
-cd build
-
-# ROCT-Thunk-Interface (hsakmt)
-if [ ! -f debs/hsakmt-roct-dev_5.5.0.99999-local_amd64.deb ]
-then
-  mkdir -p ROCT-Thunk-Interface
-  cd ROCT-Thunk-Interface
-  cmake ../../src/ROCT-Thunk-Interface
-  make -j32 package
-  cp hsakmt-roct-dev_5.5.0.99999-local_amd64.deb ../debs
-  cd ../
-fi
-
-
-# build custom LLVM
-if [ ! -f llvm-project/bin/clang ]
-then
-  mkdir -p llvm-project
-  cd llvm-project
-  cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="llvm;clang;lld" -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" ../../src/llvm-project/llvm
-  make -j32
-  cd ..
-fi
-
-# use custom LLVM
-export PATH="$PWD/llvm-project/bin:$PATH"
-
-# ROCm-Device-Libs
-if [ ! -f debs/rocm-device-libs_1.0.0.99999-local_amd64.deb ]
-then
-  mkdir -p ROCm-Device-Libs
-  cd ROCm-Device-Libs
-  cmake ../../src/ROCm-Device-Libs
-  make -j32 package
-  cp rocm-device-libs_1.0.0.99999-local_amd64.deb ../debs
-  cd ../
-fi
-
-# ROCR-Runtime
-if [ ! -f debs/hsa-rocr_1.8.0-local_amd64.deb ]
-then
-  mkdir -p ROCR-Runtime
-  cd ROCR-Runtime
-  cmake ../../src/ROCR-Runtime/src
-  make -j32 package
-  cp hsa-rocr_1.8.0-local_amd64.deb ../debs
-  cp hsa-rocr-dev_1.8.0-local_amd64.deb ../debs
-  cd ../
-fi
-
-# ROCm-OpenCL-Runtime (needs ROCclr)
-if [ ! -f debs/rocm-opencl_2.0.0-local_amd64.deb ]
-then
-  mkdir -p ROCm-OpenCL-Runtime
-  cd ROCm-OpenCL-Runtime
-  cmake ../../src/ROCm-OpenCL-Runtime
-  make -j32 package
-  cp rocm-opencl_2.0.0-local_amd64.deb ../debs
-  cp rocm-opencl-dev_2.0.0-local_amd64.deb ../debs
-  cp rocm-ocl-icd_2.0.0-local_amd64.deb ../debs
-fi
-
-# ROCm-CompilerSupport (broken)
-#mkdir -p ROCm-CompilerSupport
-#cd ROCm-CompilerSupport
-#cmake ../../src/ROCm-CompilerSupport/lib/comgr
-#make -j32

extra/assembly/rocm/rocm_setup.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-rm amdgpu-install_5.5.50500-1_all.deb
-wget https://repo.radeon.com/amdgpu-install/5.5/ubuntu/$(lsb_release -cs)/amdgpu-install_5.5.50500-1_all.deb
-sudo dpkg -i amdgpu-install_5.5.50500-1_all.deb
-sudo apt-get update
-
-# kernel driver
-sudo apt-get install amdgpu-dkms
-
-# for opencl
-sudo apt-get install rocm-opencl-runtime
-
-# for HIP
-sudo apt-get install hip-runtime-amd rocm-device-libs hip-dev

extra/assembly/rocm/sniffer/build.sh

@@ -1,11 +0,0 @@
-#!/bin/bash -e
-clang sniff.cc -Werror -shared -fPIC -I../src/ -I../src/ROCT-Thunk-Interface/include -I../src/ROCm-Device-Libs/ockl/inc -o sniff.so -lstdc++
-#AMD_LOG_LEVEL=4 HSAKMT_DEBUG_LEVEL=7 LD_PRELOAD=$PWD/sniff.so /home/tiny/build/HIP-Examples/HIP-Examples-Applications/HelloWorld/HelloWorld
-#AMD_LOG_LEVEL=4 LD_PRELOAD=$PWD/sniff.so $HOME/build/HIP-Examples/HIP-Examples-Applications/HelloWorld/HelloWorld
-#AMD_LOG_LEVEL=5 LD_PRELOAD=$PWD/sniff.so python3 ../rdna3/asm.py
-DEBUG=5 LD_PRELOAD=$PWD/sniff.so python3 ../rdna3/asm.py
-#AMD_LOG_LEVEL=5 HSAKMT_DEBUG_LEVEL=7 DEBUG=5 LD_PRELOAD=$PWD/sniff.so strace -F python3 ../rdna3/asm.py
-#LD_PRELOAD=$PWD/sniff.so python3 ../rdna3/asm.py
-#AMD_LOG_LEVEL=4 LD_PRELOAD=$PWD/sniff.so FORWARD_ONLY=1 DEBUG=2 python3 ../../../test/test_ops.py TestOps.test_add
-#AMD_LOG_LEVEL=4 HSAKMT_DEBUG_LEVEL=7 LD_PRELOAD=$PWD/sniff.so rocm-bandwidth-test -s 0 -d 1 -m 1
-#AMD_LOG_LEVEL=4 HSAKMT_DEBUG_LEVEL=7 LD_PRELOAD=$PWD/sniff.so rocm-bandwidth-test -s 1 -d 2 -m 1

extra/assembly/rocm/sniffer/sniff.cc

@@ -1,282 +0,0 @@
-// template copied from https://github.com/geohot/cuda_ioctl_sniffer/blob/master/sniff.cc
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <dlfcn.h>
-#include <signal.h>
-#include <ucontext.h>
-
-#include <sys/mman.h>
-
-// includes from the ROCm sources
-#include <linux/kfd_ioctl.h>
-#include <hsa.h>
-#include <amd_hsa_kernel_code.h>
-#include <ROCR-Runtime/src/core/inc/sdma_registers.h>
-using namespace rocr::AMD;
-
-#include <string>
-#include <map>
-std::map<int, std::string> files;
-std::map<uint64_t, uint64_t> ring_base_addresses;
-
-#define D(args...) fprintf(stderr, args)
-
-uint64_t doorbell_offset = -1;
-std::map<uint64_t, int> queue_types;
-
-void hexdump(void *d, int l) {
-  for (int i = 0; i < l; i++) {
-    if (i%0x10 == 0 && i != 0) printf("\n");
-    if (i%0x10 == 8) printf(" ");
-    if (i%0x10 == 0) printf("%8X: ", i);
-    printf("%2.2X ", ((uint8_t*)d)[i]);
-  }
-  printf("\n");
-}
-
-extern "C" {
-
-// https://defuse.ca/online-x86-assembler.htm#disassembly2
-static void handler(int sig, siginfo_t *si, void *unused) {
-  ucontext_t *u = (ucontext_t *)unused;
-  uint8_t *rip = (uint8_t*)u->uc_mcontext.gregs[REG_RIP];
-
-  int store_size = 0;
-  uint64_t value;
-  if (rip[0] == 0x48 && rip[1] == 0x89 && rip[2] == 0x30) {
-    // 0:  48 89 30                mov    QWORD PTR [rax],rsi
-    store_size = 8;
-    value = u->uc_mcontext.gregs[REG_RSI];
-    u->uc_mcontext.gregs[REG_RIP] += 3;
-  } else if (rip[0] == 0x4c && rip[1] == 0x89 && rip[2] == 0x28) {
-    // 0:  4c 89 28                mov    QWORD PTR [rax],r13
-    store_size = 8;
-    value = u->uc_mcontext.gregs[REG_R13];
-    u->uc_mcontext.gregs[REG_RIP] += 3;
-  } else {
-    D("segfault %02X %02X %02X %02X %02X %02X %02X %02X rip: %p addr: %p\n", rip[0], rip[1], rip[2], rip[3], rip[4], rip[5], rip[6], rip[7], rip, si->si_addr);
-    D("rax: %llx rcx: %llx rdx: %llx rsi: %llx rbx: %llx\n", u->uc_mcontext.gregs[REG_RAX], u->uc_mcontext.gregs[REG_RCX], u->uc_mcontext.gregs[REG_RDX], u->uc_mcontext.gregs[REG_RSI], u->uc_mcontext.gregs[REG_RBX]);
-    exit(-1);
-  }
-
-  uint64_t ring_base_address = ring_base_addresses[((uint64_t)si->si_addr)&0xFFF];
-  int queue_type = queue_types[((uint64_t)si->si_addr)&0xFFF];
-  D("%16p: \u001b[31mDING DONG\u001b[0m (queue_type %d) store(%d): 0x%8lx -> %p ring_base_address:0x%lx\n", rip, queue_type, store_size, value, si->si_addr, ring_base_address);
-
-  if (queue_type == KFD_IOC_QUEUE_TYPE_SDMA) {
-    uint8_t *sdma_ptr = (uint8_t*)(ring_base_address);
-    while (sdma_ptr < ((uint8_t*)(ring_base_address)+value)) {
-      D("0x%3lx: ", sdma_ptr-(uint8_t*)(ring_base_address));
-      if (sdma_ptr[0] == SDMA_OP_TIMESTAMP) {
-        D("SDMA_PKT_TIMESTAMP\n");
-        sdma_ptr += sizeof(SDMA_PKT_TIMESTAMP);
-      } else if (sdma_ptr[0] == SDMA_OP_GCR) {
-        D("SDMA_PKT_GCR\n");
-        sdma_ptr += sizeof(SDMA_PKT_GCR);
-      } else if (sdma_ptr[0] == SDMA_OP_ATOMIC) {
-        D("SDMA_PKT_ATOMIC\n");
-        sdma_ptr += sizeof(SDMA_PKT_ATOMIC);
-      } else if (sdma_ptr[0] == SDMA_OP_FENCE) {
-        D("SDMA_PKT_FENCE\n");
-        sdma_ptr += sizeof(SDMA_PKT_FENCE);
-      } else if (sdma_ptr[0] == SDMA_OP_TRAP) {
-        D("SDMA_PKT_TRAP\n");
-        sdma_ptr += sizeof(SDMA_PKT_TRAP);
-      } else if (sdma_ptr[0] == SDMA_OP_COPY && sdma_ptr[1] == SDMA_SUBOP_COPY_LINEAR) {
-        SDMA_PKT_COPY_LINEAR *pkt = (SDMA_PKT_COPY_LINEAR *)sdma_ptr;
-        D("SDMA_PKT_COPY_LINEAR: count:0x%x src:0x%lx dst:0x%lx\n", pkt->COUNT_UNION.count+1,
-          (uint64_t)pkt->SRC_ADDR_LO_UNION.src_addr_31_0 | ((uint64_t)pkt->SRC_ADDR_HI_UNION.src_addr_63_32 << 32),
-          (uint64_t)pkt->DST_ADDR_LO_UNION.dst_addr_31_0 | ((uint64_t)pkt->DST_ADDR_HI_UNION.dst_addr_63_32 << 32)
-        );
-        sdma_ptr += sizeof(SDMA_PKT_COPY_LINEAR);
-      } else {
-        D("unhandled packet type %d %d, exiting\n", sdma_ptr[0], sdma_ptr[1]);
-        break;
-      }
-    }
-
-    //hexdump((void*)(ring_base_address), 0x100);
-  } else if (queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) {
-    hsa_kernel_dispatch_packet_t *pkt = (hsa_kernel_dispatch_packet_t *)(ring_base_address+value*0x40);
-    if ((pkt->header&0xFF) == HSA_PACKET_TYPE_KERNEL_DISPATCH) {
-      D("HSA_PACKET_TYPE_KERNEL_DISPATCH -- setup:%d workgroup[%d, %d, %d] grid[%d, %d, %d] kernel_object:0x%lx kernarg_address:%p\n", pkt->setup, pkt->workgroup_size_x, pkt->workgroup_size_y, pkt->workgroup_size_z, pkt->grid_size_x, pkt->grid_size_y, pkt->grid_size_z, pkt->kernel_object, pkt->kernarg_address);
-      amd_kernel_code_t *code = (amd_kernel_code_t *)pkt->kernel_object;
-      D("kernel_code_entry_byte_offset:%lx\n", code->kernel_code_entry_byte_offset);
-      uint32_t *kernel_code = (uint32_t*)(pkt->kernel_object + code->kernel_code_entry_byte_offset);
-      int code_len = 0;
-      while (kernel_code[code_len] != 0xbf9f0000 && kernel_code[code_len] != 0) code_len++;
-      hexdump(kernel_code, code_len*4);
-      /*FILE *f = fopen("/tmp/kernel_code", "wb");
-      fwrite(kernel_code, 4, code_len, f);
-      fclose(f);
-      system("python -c 'print(\" \".join([(\"0x%02X\"%x) for x in open(\"/tmp/kernel_code\", \"rb\").read()]))' | ../build/llvm-project/bin/llvm-mc --disassemble --arch=amdgcn --mcpu=gfx1100 --show-encoding");*/
-      D("kernargs (kernarg_segment_byte_size:0x%lx)\n", code->kernarg_segment_byte_size);
-      // get length
-      int i;
-      for (i = 0; i < 0x400; i+=0x10) {
-        if (memcmp((void*)((uint64_t)pkt->kernarg_address+i), "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0x10) == 0) break;
-      }
-      hexdump((void*)pkt->kernarg_address, i+0x10);
-    } else if ((pkt->header&0xFF) == HSA_PACKET_TYPE_BARRIER_AND) {
-      hsa_barrier_and_packet_t *pkt_and = (hsa_barrier_and_packet_t *)(ring_base_address+value*0x40);
-      D("HSA_PACKET_TYPE_BARRIER_AND completion_signal:0x%lx\n", pkt_and->completion_signal.handle);
-      //hexdump((void*)(ring_base_address+value*0x40), 0x40);
-    } else if ((pkt->header&0xFF) == HSA_PACKET_TYPE_VENDOR_SPECIFIC) {
-      D("HSA_PACKET_TYPE_VENDOR_SPECIFIC\n");
-      hexdump((void*)(ring_base_address+value*0x40), 0x40);
-    } else {
-      hexdump((void*)(ring_base_address+value*0x40), 0x40);
-    }
-  }
-
-  mprotect((void *)((uint64_t)si->si_addr & ~0xFFF), 0x2000, PROT_READ | PROT_WRITE);
-  if (store_size == 8) {
-    *(volatile uint64_t*)(si->si_addr) = value;
-  } else if (store_size == 4) {
-    *(volatile uint32_t*)(si->si_addr) = value;
-  } else if (store_size == 2) {
-    *(volatile uint16_t*)(si->si_addr) = value;
-  } else {
-    D("store size not supported\n");
-    exit(-1);
-  }
-  mprotect((void *)((uint64_t)si->si_addr & ~0xFFF), 0x2000, PROT_NONE);
-}
-
-void register_sigsegv_handler() {
-  struct sigaction sa = {0};
-  sa.sa_flags = SA_SIGINFO;
-  sigemptyset(&sa.sa_mask);
-  sa.sa_sigaction = handler;
-  if (sigaction(SIGSEGV, &sa, NULL) == -1) {
-    D("ERROR: failed to register sigsegv handler");
-    exit(-1);
-  }
-  // NOTE: python (or ocl runtime?) blocks the SIGSEGV signal
-  sigset_t x;
-  sigemptyset(&x);
-  sigaddset(&x, SIGSEGV);
-  sigprocmask(SIG_UNBLOCK, &x, NULL);
-}
-
-int (*my_open)(const char *pathname, int flags, mode_t mode);
-#undef open
-int open(const char *pathname, int flags, mode_t mode) {
-  if (my_open == NULL) my_open = reinterpret_cast<decltype(my_open)>(dlsym(RTLD_NEXT, "open"));
-  int ret = my_open(pathname, flags, mode);
-  //D("open %s (0o%o) = %d\n", pathname, flags, ret);
-  files[ret] = pathname;
-  return ret;
-}
-
-
-int (*my_open64)(const char *pathname, int flags, mode_t mode);
-#undef open
-int open64(const char *pathname, int flags, mode_t mode) {
-  if (my_open64 == NULL) my_open64 = reinterpret_cast<decltype(my_open64)>(dlsym(RTLD_NEXT, "open64"));
-  int ret = my_open64(pathname, flags, mode);
-  //D("open %s (0o%o) = %d\n", pathname, flags, ret);
-  files[ret] = pathname;
-  return ret;
-}
-
-void *(*my_mmap)(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
-#undef mmap
-void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
-  if (my_mmap == NULL) my_mmap = reinterpret_cast<decltype(my_mmap)>(dlsym(RTLD_NEXT, "mmap"));
-  void *ret = my_mmap(addr, length, prot, flags, fd, offset);
-
-  if (doorbell_offset != -1 && offset == doorbell_offset) {
-    D("HIDDEN DOORBELL %p, handled by %p\n", addr, handler);
-    register_sigsegv_handler();
-    mprotect(addr, length, PROT_NONE);
-  }
-
-  if (fd != -1) D("mmapped %p (target %p) with flags 0x%x length 0x%zx fd %d %s offset 0x%lx\n", ret, addr, flags, length, fd, files[fd].c_str(), offset);
-  return ret;
-}
-
-void *(*my_mmap64)(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
-#undef mmap64
-void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { return mmap(addr, length, prot, flags, fd, offset); }
-
-int ioctl_num = 1;
-int (*my_ioctl)(int filedes, unsigned long request, void *argp) = NULL;
-#undef ioctl
-int ioctl(int filedes, unsigned long request, void *argp) {
-  if (my_ioctl == NULL) my_ioctl = reinterpret_cast<decltype(my_ioctl)>(dlsym(RTLD_NEXT, "ioctl"));
-  int ret = 0;
-  ret = my_ioctl(filedes, request, argp);
-  if (!files.count(filedes)) return ret;
-
-  uint8_t type = (request >> 8) & 0xFF;
-  uint8_t nr = (request >> 0) & 0xFF;
-  uint16_t size = (request >> 16) & 0xFFF;
-
-  D("%3d: %d = %3d(%20s) 0x%3x ", ioctl_num, ret, filedes, files[filedes].c_str(), size);
-
-  if (request == AMDKFD_IOC_SET_EVENT) {
-    kfd_ioctl_set_event_args *args = (kfd_ioctl_set_event_args *)argp;
-    D("AMDKFD_IOC_SET_EVENT event_id:%d", args->event_id);
-  } else if (request == AMDKFD_IOC_ALLOC_MEMORY_OF_GPU) {
-    kfd_ioctl_alloc_memory_of_gpu_args *args = (kfd_ioctl_alloc_memory_of_gpu_args *)argp;
-    D("AMDKFD_IOC_ALLOC_MEMORY_OF_GPU va_addr:0x%llx size:0x%llx handle:%llX gpu_id:0x%x", args->va_addr, args->size, args->handle, args->gpu_id);
-  } else if (request == AMDKFD_IOC_MAP_MEMORY_TO_GPU) {
-    kfd_ioctl_map_memory_to_gpu_args *args = (kfd_ioctl_map_memory_to_gpu_args *)argp;
-    D("AMDKFD_IOC_MAP_MEMORY_TO_GPU handle:%llX", args->handle);
-  } else if (request == AMDKFD_IOC_CREATE_EVENT) {
-    kfd_ioctl_create_event_args *args = (kfd_ioctl_create_event_args *)argp;
-    D("AMDKFD_IOC_CREATE_EVENT event_page_offset:0x%llx event_type:%d event_id:%d", args->event_page_offset, args->event_type, args->event_id);
-  } else if (request == AMDKFD_IOC_WAIT_EVENTS) {
-    D("AMDKFD_IOC_WAIT_EVENTS");
-  } else if (request == AMDKFD_IOC_SET_XNACK_MODE) {
-    D("AMDKFD_IOC_SET_XNACK_MODE");
-  } else if (request == AMDKFD_IOC_SVM || (type == 0x4b && nr == 0x20)) {
-    // NOTE: this one is variable length
-    kfd_ioctl_svm_args *args = (kfd_ioctl_svm_args *)argp;
-    D("AMDKFD_IOC_SVM start_addr:0x%llx size:0x%llx op:%d", args->start_addr, args->size, args->op);
-  } else if (request == AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU) {
-    kfd_ioctl_unmap_memory_from_gpu_args *args = (kfd_ioctl_unmap_memory_from_gpu_args *)argp;
-    D("AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU handle:%llX", args->handle);
-  } else if (request == AMDKFD_IOC_FREE_MEMORY_OF_GPU) {
-    D("AMDKFD_IOC_FREE_MEMORY_OF_GPU");
-  } else if (request == AMDKFD_IOC_SET_SCRATCH_BACKING_VA) {
-    D("AMDKFD_IOC_SET_SCRATCH_BACKING_VA");
-  } else if (request == AMDKFD_IOC_GET_TILE_CONFIG) {
-    D("AMDKFD_IOC_GET_TILE_CONFIG");
-  } else if (request == AMDKFD_IOC_SET_TRAP_HANDLER) {
-    D("AMDKFD_IOC_SET_TRAP_HANDLER");
-  } else if (request == AMDKFD_IOC_GET_VERSION) {
-    kfd_ioctl_get_version_args *args = (kfd_ioctl_get_version_args *)argp;
-    D("AMDKFD_IOC_GET_VERSION major_version:%d minor_version:%d", args->major_version, args->minor_version);
-  } else if (request == AMDKFD_IOC_GET_PROCESS_APERTURES_NEW) {
-    D("AMDKFD_IOC_GET_PROCESS_APERTURES_NEW");
-  } else if (request == AMDKFD_IOC_ACQUIRE_VM) {
-    D("AMDKFD_IOC_ACQUIRE_VM");
-  } else if (request == AMDKFD_IOC_SET_MEMORY_POLICY) {
-    D("AMDKFD_IOC_SET_MEMORY_POLICY");
-  } else if (request == AMDKFD_IOC_GET_CLOCK_COUNTERS) {
-    D("AMDKFD_IOC_GET_CLOCK_COUNTERS");
-  } else if (request == AMDKFD_IOC_CREATE_QUEUE) {
-    kfd_ioctl_create_queue_args *args = (kfd_ioctl_create_queue_args *)argp;
-    D("AMDKFD_IOC_CREATE_QUEUE\n");
-    D("queue_type:%d ring_base_address:0x%llx\n", args->queue_type, args->ring_base_address);
-    D("eop_buffer_address:0x%llx ctx_save_restore_address:0x%llx\n", args->eop_buffer_address, args->ctx_save_restore_address);
-    D("ring_size:0x%x queue_priority:%d\n", args->ring_size, args->queue_priority);
-    D("RETURNS write_pointer_address:0x%llx read_pointer_address:0x%llx doorbell_offset:0x%llx queue_id:%d\n", args->write_pointer_address, args->read_pointer_address, args->doorbell_offset, args->queue_id);
-    //D("RETURNS *write_pointer_address:0x%llx *read_pointer_address:0x%llx\n", *(uint64_t*)args->write_pointer_address, *(uint64_t*)args->read_pointer_address);
-    ring_base_addresses[args->doorbell_offset&0xFFF] = args->ring_base_address;
-    queue_types[args->doorbell_offset&0xFFF] = args->queue_type;
-    doorbell_offset = args->doorbell_offset&~0xFFF;
-  } else {
-    D("type:0x%x nr:0x%x size:0x%x", type, nr, size);
-  }
-
-  D("\n");
-  ioctl_num++;
-  return ret;
-}
-
-}

extra/gemm/.gitignore

@@ -1,3 +1,2 @@
-*.s
 *.ll
 fp32_sgemm_amd

extra/gemm/asm/test.py

@@ -0,0 +1,72 @@
+# Run assembly on the AMD runtime and check correctness
+# VIZ=2 to profile
+import pathlib
+from tinygrad import Tensor, Device, dtypes
+from tinygrad.engine.realize import ExecItem, CompiledRunner
+from tinygrad.renderer import ProgramSpec
+from tinygrad.uop.ops import track_rewrites, UOp
+from tinygrad.helpers import TracingKey, getenv
+
+fp = pathlib.Path(__file__).parent/"gemm.s"
+
+N = getenv("N", 8192)
+THREADS_PER_WG = 256
+NUM_WG = N//THREADS_PER_WG * N//THREADS_PER_WG
+
+assert N % THREADS_PER_WG == 0, "N must be divisible by THREADS_PER_WG"
+
+# ** generate inputs on CPU
+
+scale = 10.0
+
+import torch
+torch.manual_seed(0)
+A = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
+B = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
+Bt = B.t().contiguous() # transpose B for the baseline gemm
+C_torch = A@Bt
+
+# ** copy buffers to AMD
+
+# input creation and validation run on the copy engine for simpler tracing
+
+def from_torch(t:torch.Tensor) -> Tensor:
+  return Tensor.from_blob(t.data_ptr(), t.shape, dtype=dtypes.bfloat16, device="cpu").to(Device.DEFAULT).realize()
+
+C_tiny = Tensor.matmul(from_torch(A), from_torch(Bt), dtype=dtypes.float32).cast(dtypes.bfloat16)
+C_asm = Tensor.empty_like(C_tiny)
+C_asm.uop.buffer.allocate()
+
+# ** run gemms
+
+# baseline tinygrad
+sched = C_tiny.schedule()
+assert len(sched) == 1
+eis:list[ExecItem] = [sched[-1].lower()]
+ast = sched[-1].ast
+
+# assembly gemm
+@track_rewrites(name=lambda ret: TracingKey(ret.name, (ret.function_name,), ret))
+def get_asm_prg() -> ProgramSpec:
+  src = fp.read_text()
+  lib = Device[Device.DEFAULT].compiler.compile(src)
+  return ProgramSpec("gemm", src, Device.DEFAULT, ast, lib=lib, global_size=[NUM_WG, 1, 1], local_size=[THREADS_PER_WG, 1, 1],
+                     globals=[0, 1, 2], vars=[UOp.variable("SZ", 256, 8192), UOp.variable("NUM_WG", 1, 1024)])
+eis.append(ExecItem(ast, [C_asm.uop.buffer, from_torch(B).uop.buffer, from_torch(A).uop.buffer], fixedvars={"SZ":N, "NUM_WG":NUM_WG},
+                    prg=CompiledRunner(get_asm_prg())))
+
+for ei in eis:
+  et = ei.run(wait=True)
+  print(f"{(N*N*N*2 / et)*1e-12:.2f} REAL TFLOPS")
+
+# ** correctness
+
+import ctypes
+
+def torch_bf16(t:Tensor) -> torch.tensor:
+  asm_out = t.to("cpu").realize().uop.buffer._buf
+  buf = (ctypes.c_uint16*C_asm.uop.size).from_address(asm_out.va_addr)
+  return torch.frombuffer(buf, dtype=torch.bfloat16, count=C_asm.uop.size).reshape(C_asm.shape)
+
+assert torch.allclose(torch_bf16(C_asm), C_torch, rtol=1e-2, atol=1e-3)
+assert torch.allclose(torch_bf16(C_tiny), C_torch, rtol=1e-2, atol=1e-3)

extra/gemm/asm/unpack_kd.py

@@ -0,0 +1,179 @@
+# unpack the complete kernel descriptor of an amdgpu ELF of for gfx950
+# https://rocm.docs.amd.com/projects/llvm-project/en/latest/LLVM/llvm/html/AMDGPUUsage.html#code-object-v3-kernel-descriptor
+import struct, pathlib
+from tinygrad.runtime.support.elf import elf_loader
+
+def bits(x, lo, hi): return (x >> lo) & ((1 << (hi - lo + 1)) - 1)
+def assert_zero(x, lo, hi): assert bits(x, lo, hi) == 0
+
+with open(fp:=pathlib.Path(__file__).parent/"lib", "rb") as f:
+  lib = f.read()
+
+image, sections, relocs = elf_loader(lib)
+rodata_entry = next((sh.header.sh_addr for sh in sections if sh.name == ".rodata"))
+
+# rodata is exactly 64 bytes
+kd = image[rodata_entry:rodata_entry+64]
+desc = int.from_bytes(kd, byteorder="little")
+
+group_segment_fixed_size = bits(desc, 0, 31)
+private_segment_fixed_size = bits(desc, 32, 63)
+kernarg_size = bits(desc, 64, 95)
+reserved_127_96 = bits(desc, 96, 127)
+assert reserved_127_96 == 0
+
+print("GROUP_SEGMENT_FIXED_SIZE:", group_segment_fixed_size)
+print("PRIVATE_SEGMENT_FIXED_SIZE:", private_segment_fixed_size)
+print("KERNARG_SIZE:", kernarg_size)
+print("RESERVED 127:96:", reserved_127_96)
+
+entry_off = bits(desc, 128, 191)
+
+# sign-extend manually if needed
+if entry_off & (1 << 63):
+  entry_off -= 1 << 64
+
+print("KERNEL_CODE_ENTRY_BYTE_OFFSET:", entry_off)
+
+kd_addr = 0x1840
+entry_addr = kd_addr + entry_off
+
+print("Computed entry address: 0x%016x" % entry_addr)
+print("256B aligned:", entry_addr % 256 == 0)
+
+pgm_rsrc3 = bits(desc, 352, 383)
+pgm_rsrc1 = bits(desc, 384, 415)
+pgm_rsrc2 = bits(desc, 416, 447)
+
+print("COMPUTE_PGM_RSRC3: 0x%08x" % pgm_rsrc3)
+print("COMPUTE_PGM_RSRC1: 0x%08x" % pgm_rsrc1)
+print("COMPUTE_PGM_RSRC2: 0x%08x" % pgm_rsrc2)
+
+# rsrc 3
+
+accum_offset_raw = bits(pgm_rsrc3, 0, 5)
+assert_zero(pgm_rsrc3, 6, 15)
+tg_split = bits(pgm_rsrc3, 16, 16)
+accum_offset_vgprs = (accum_offset_raw + 1) * 4
+print("RSRC3.ACCUM_OFFSET (AccVGPR index):", accum_offset_vgprs)
+print("RSRC3.TG_SPLIT:", tg_split)
+
+# rsrc 1
+
+vgpr_gran = bits(pgm_rsrc1, 0, 5)
+sgpr_gran = bits(pgm_rsrc1, 6, 9)
+assert_zero(pgm_rsrc1, 27, 28)
+
+# NOTE: this is vgprs + agprs
+vgprs_used = (vgpr_gran + 1) * 8
+assert 0 <= vgprs_used <= 512
+
+k = sgpr_gran // 2
+sgprs_used = (k + 1) * 16
+
+print("RSRC1.VGPRS:", vgprs_used)
+print("RSRC1.SGPRS:", sgprs_used)
+
+assert_zero(pgm_rsrc1, 10, 11)
+
+float_round_mode_32 = bits(pgm_rsrc1, 12, 13)
+float_round_mode_16_64 = bits(pgm_rsrc1, 15, 14)
+float_denorm_mode_32 = bits(pgm_rsrc1, 16, 17)
+float_denorm_mode_16_64 = bits(pgm_rsrc1, 18, 19)
+
+priv = bits(pgm_rsrc1, 20, 20)
+assert priv == 0
+enable_dx10_clamp_wg_rr_en = bits(pgm_rsrc1, 21, 21)
+debug_mode = bits(pgm_rsrc1, 22, 22)
+enable_ieee_mode = bits(pgm_rsrc1, 23, 23)
+bulky = bits(pgm_rsrc1, 24, 24)
+assert bulky == 0
+cdbg_user = bits(pgm_rsrc1, 25, 25)
+assert cdbg_user == 0
+fp16_ovfl = bits(pgm_rsrc1, 26, 26)
+assert_zero(pgm_rsrc1, 27, 28)  # reserved
+assert_zero(pgm_rsrc1, 29, 29)  # WGP_MODE (reserved on gfx9)
+assert_zero(pgm_rsrc1, 30, 30)  # MEM_ORDERED (reserved on gfx9)
+assert_zero(pgm_rsrc1, 31, 31)  # FWD_PROGRESS (reserved on gfx9)
+
+# rsrc 2
+
+enable_private_segment = bits(pgm_rsrc2, 0, 0)  # SCRATCH_EN
+user_sgpr_count = bits(pgm_rsrc2, 1, 5)         # USER_SGPR
+enable_trap_handler = bits(pgm_rsrc2, 6, 6)     # TRAP_PRESENT (must be 0 here)
+assert enable_trap_handler == 0
+
+enable_sgpr_workgroup_id_x = bits(pgm_rsrc2, 7, 7)
+enable_sgpr_workgroup_id_y = bits(pgm_rsrc2, 8, 8)
+enable_sgpr_workgroup_id_z = bits(pgm_rsrc2, 9, 9)
+enable_sgpr_workgroup_info = bits(pgm_rsrc2, 10, 10)
+
+enable_vgpr_workitem_id = bits(pgm_rsrc2, 11, 12)  # TIDIG_CMP_CNT enum (0..3)
+
+enable_exception_address_watch = bits(pgm_rsrc2, 13, 13)
+assert enable_exception_address_watch == 0
+enable_exception_memory = bits(pgm_rsrc2, 14, 14)
+assert enable_exception_memory == 0
+
+granulated_lds_size = bits(pgm_rsrc2, 15, 23)
+assert granulated_lds_size == 0  # spec: must be 0; CP uses dispatch packet rounding
+
+enable_exception_fp_invalid = bits(pgm_rsrc2, 24, 24)
+enable_exception_fp_denorm_src = bits(pgm_rsrc2, 25, 25)
+enable_exception_fp_div0 = bits(pgm_rsrc2, 26, 26)
+enable_exception_fp_overflow = bits(pgm_rsrc2, 27, 27)
+enable_exception_fp_underflow = bits(pgm_rsrc2, 28, 28)
+enable_exception_fp_inexact = bits(pgm_rsrc2, 29, 29)
+enable_exception_int_div0 = bits(pgm_rsrc2, 30, 30)
+
+assert_zero(pgm_rsrc2, 31, 31)
+
+print("RSRC2.ENABLE_PRIVATE_SEGMENT:", enable_private_segment)
+print("RSRC2.USER_SGPR_COUNT:", user_sgpr_count)
+print("RSRC2.ENABLE_SGPR_WORKGROUP_ID_X:", enable_sgpr_workgroup_id_x)
+print("RSRC2.ENABLE_SGPR_WORKGROUP_ID_Y:", enable_sgpr_workgroup_id_y)
+print("RSRC2.ENABLE_SGPR_WORKGROUP_ID_Z:", enable_sgpr_workgroup_id_z)
+print("RSRC2.ENABLE_SGPR_WORKGROUP_INFO:", enable_sgpr_workgroup_info)
+print("RSRC2.ENABLE_VGPR_WORKITEM_ID (enum):", enable_vgpr_workitem_id)
+
+print("RSRC2.EXC_FP_INVALID:", enable_exception_fp_invalid)
+print("RSRC2.EXC_FP_DENORM_SRC:", enable_exception_fp_denorm_src)
+print("RSRC2.EXC_FP_DIV0:", enable_exception_fp_div0)
+print("RSRC2.EXC_FP_OVERFLOW:", enable_exception_fp_overflow)
+print("RSRC2.EXC_FP_UNDERFLOW:", enable_exception_fp_underflow)
+print("RSRC2.EXC_FP_INEXACT:", enable_exception_fp_inexact)
+print("RSRC2.EXC_INT_DIV0:", enable_exception_int_div0)
+
+# user sgprs
+
+enable_sgpr_private_segment_buffer = bits(desc, 448, 448)
+enable_sgpr_dispatch_ptr = bits(desc, 449, 449)
+enable_sgpr_queue_ptr = bits(desc, 450, 450)
+enable_sgpr_kernarg_segment_ptr = bits(desc, 451, 451)
+enable_sgpr_dispatch_id = bits(desc, 452, 452)
+enable_sgpr_flat_scratch_init = bits(desc, 453, 453)
+enable_sgpr_private_segment_size = bits(desc, 454, 454)
+
+assert_zero(desc, 455, 457)
+
+print("DESC.ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER:", enable_sgpr_private_segment_buffer)
+print("DESC.ENABLE_SGPR_DISPATCH_PTR:", enable_sgpr_dispatch_ptr)
+print("DESC.ENABLE_SGPR_QUEUE_PTR:", enable_sgpr_queue_ptr)
+print("DESC.ENABLE_SGPR_KERNARG_SEGMENT_PTR:", enable_sgpr_kernarg_segment_ptr)
+print("DESC.ENABLE_SGPR_DISPATCH_ID:", enable_sgpr_dispatch_id)
+print("DESC.ENABLE_SGPR_FLAT_SCRATCH_INIT:", enable_sgpr_flat_scratch_init)
+print("DESC.ENABLE_SGPR_PRIVATE_SEGMENT_SIZE:", enable_sgpr_private_segment_size)
+
+assert_zero(desc, 458, 459)
+
+uses_dynamic_stack = bits(desc, 459, 460)
+print("DESC.USES_DYNAMIC_STACK:", uses_dynamic_stack)
+
+assert_zero(desc, 460, 463)
+kernarg_preload_spec_length = bits(desc, 464, 470)
+print("DESC.KERNARG_PRELOAD_SPEC_LENGTH:", kernarg_preload_spec_length)
+
+kernarg_preload_spec_offset = bits(desc, 471, 479)
+print("DESC.KERNARG_PRELOAD_SPEC_OFFSET:", kernarg_preload_spec_offset)
+
+assert_zero(desc, 480, 511)

extra/models/llama.py

@@ -245,6 +245,11 @@ def convert_from_huggingface(weights:dict[str, Tensor], n_layers: int, n_heads:
       continue
     sd[keymap[k]] = v
   for k,v in experts.items(): sd[k] = Tensor.stack(*[v[i] for i in range(len(v))])
+
+  # Handle tied embeddings (e.g., Llama 3.2 1B Instruct where lm_head shares weights with embed_tokens)
+  if "output.weight" not in sd and "tok_embeddings.weight" in sd:
+    sd["output.weight"] = sd["tok_embeddings.weight"]
+
   return sd
 
 def convert_from_gguf(weights:dict[str, Tensor], n_layers:int):

extra/optimization/extract_policynet.py

@@ -1,114 +0,0 @@
-import os, sys, sqlite3, pickle, random
-from tqdm import tqdm, trange
-from copy import deepcopy
-from tinygrad.nn import Linear
-from tinygrad.tensor import Tensor
-from tinygrad.nn.optim import Adam
-from tinygrad.nn.state import get_parameters, get_state_dict, safe_save, safe_load, load_state_dict
-from tinygrad.codegen.opt.search import actions
-from extra.optimization.helpers import load_worlds, ast_str_to_lin, lin_to_feats, assert_same_lin
-from tinygrad.codegen.opt.kernel import Kernel
-from tinygrad.helpers import getenv
-
-# stuff needed to unpack a kernel
-from tinygrad.uop.ops import LazyOp, TernaryOps, BinaryOps, UnaryOps, ReduceOps, BufferOps, MemBuffer, ConstBuffer
-from tinygrad.dtype import dtypes
-from tinygrad.shape.shapetracker import ShapeTracker
-from tinygrad.shape.view import View
-from tinygrad.uop.ops import Variable
-inf, nan = float('inf'), float('nan')
-from tinygrad.codegen.opt.kernel import Opt, OptOps
-
-INNER = 256
-class PolicyNet:
-  def __init__(self):
-    self.l1 = Linear(1021,INNER)
-    self.l2 = Linear(INNER,INNER)
-    self.l3 = Linear(INNER,1+len(actions))
-  def __call__(self, x):
-    x = self.l1(x).relu()
-    x = self.l2(x).relu().dropout(0.9)
-    return self.l3(x).log_softmax()
-
-def dataset_from_cache(fn):
-  conn = sqlite3.connect(fn)
-  cur = conn.cursor()
-  cur.execute("SELECT * FROM beam_search")
-  X,A = [], []
-  for f in tqdm(cur.fetchall()):
-    Xs,As = [], []
-    try:
-      lin = Kernel(eval(f[0]))
-      opts = pickle.loads(f[-1])
-      for o in opts:
-        Xs.append(lin_to_feats(lin, use_sts=True))
-        As.append(actions.index(o))
-        lin.apply_opt(o)
-      Xs.append(lin_to_feats(lin, use_sts=True))
-      As.append(0)
-    except Exception:
-      pass
-    X += Xs
-    A += As
-  return X,A
-
-if __name__ == "__main__":
-  if getenv("REGEN"):
-    X,V = dataset_from_cache(sys.argv[1] if len(sys.argv) > 1 else "/tmp/tinygrad_cache")
-    safe_save({"X": Tensor(X), "V": Tensor(V)}, "/tmp/dataset_policy")
-  else:
-    ld = safe_load("/tmp/dataset_policy")
-    X,V = ld['X'].numpy(), ld['V'].numpy()
-
-  print(X.shape, V.shape)
-  order = list(range(X.shape[0]))
-  random.shuffle(order)
-  X, V = X[order], V[order]
-
-  ratio = -256
-  X_test, V_test = Tensor(X[ratio:]), Tensor(V[ratio:])
-  X,V = X[:ratio], V[:ratio]
-  print(X.shape, V.shape)
-
-  net = PolicyNet()
-  #if os.path.isfile("/tmp/policynet.safetensors"): load_state_dict(net, safe_load("/tmp/policynet.safetensors"))
-  optim = Adam(get_parameters(net))
-
-  def get_minibatch(X,Y,bs):
-    xs, ys = [], []
-    for _ in range(bs):
-      sel = random.randint(0, len(X)-1)
-      xs.append(X[sel])
-      ys.append(Y[sel])
-    return Tensor(xs), Tensor(ys)
-
-  Tensor.training = True
-  losses = []
-  test_losses = []
-  test_accuracy = 0
-  test_loss = float('inf')
-  for i in (t:=trange(500)):
-    x,y = get_minibatch(X,V,bs=256)
-    out = net(x)
-    loss = out.sparse_categorical_crossentropy(y)
-    optim.zero_grad()
-    loss.backward()
-    optim.step()
-    cat = out.argmax(axis=-1)
-    accuracy = (cat == y).mean()
-    t.set_description(f"loss {loss.numpy():7.2f} accuracy {accuracy.numpy()*100:7.2f}%, test loss {test_loss:7.2f} test accuracy {test_accuracy*100:7.2f}%")
-
-    losses.append(loss.numpy().item())
-    test_losses.append(test_loss)
-    if i % 10:
-      out = net(X_test)
-      test_loss = out.sparse_categorical_crossentropy(V_test).square().mean().numpy().item()
-      cat = out.argmax(axis=-1)
-      test_accuracy = (cat == y).mean().numpy()
-
-  safe_save(get_state_dict(net), "/tmp/policynet.safetensors")
-
-  import matplotlib.pyplot as plt
-  plt.plot(losses[10:])
-  plt.plot(test_losses[10:])
-  plt.show()

extra/optimization/extract_sa_pairs.py

@@ -1,129 +0,0 @@
-import sys, sqlite3, pickle, math
-from collections import defaultdict
-from tqdm import tqdm, trange
-import numpy as np
-
-# stuff needed to unpack a kernel
-from tinygrad.uop.ops import LazyOp, TernaryOps, BinaryOps, UnaryOps, ReduceOps, BufferOps, MemBuffer, ConstBuffer
-from tinygrad.dtype import dtypes
-from tinygrad.shape.shapetracker import ShapeTracker
-from tinygrad.shape.view import View
-from tinygrad.uop.ops import Variable
-inf, nan = float('inf'), float('nan')
-from tinygrad.codegen.opt.kernel import Opt, OptOps
-
-# more stuff
-from tinygrad.codegen.opt.kernel import Kernel
-from tinygrad.codegen.opt.search import actions
-from extra.optimization.helpers import lin_to_feats
-from extra.optimization.pretrain_valuenet import ValueNet
-from tinygrad.nn.optim import Adam
-from tinygrad.nn.state import get_parameters, get_state_dict, safe_save, safe_load, load_state_dict
-import random
-from tinygrad.tensor import Tensor
-from tinygrad.helpers import getenv
-
-def dataset_from_cache(fn):
-  conn = sqlite3.connect(fn)
-  cur = conn.cursor()
-  cur.execute("SELECT * FROM time_linearizer")
-  grouped = defaultdict(dict)
-  for f in tqdm(cur.fetchall()): grouped[f[0]][f[1:-1]] = pickle.loads(f[-1])
-
-  opts_to_outcome = {}
-
-  for ast,sk in grouped.items():
-    cnts = defaultdict(int)
-    for sks,tm in sk.items():
-      if sks[1] != 1: continue
-      opts = eval(sks[0])
-      cnts[(len(opts), sks[1])] += 1
-      opts_to_outcome[(ast, tuple(opts))] = tm
-    #print(cnts)
-
-  S,A,V = [], [], []
-  for ast,k in tqdm(opts_to_outcome):
-    if len(k) == 0: continue
-    old_tm = min(opts_to_outcome[(ast,k[:-1])])
-    new_tm = min(opts_to_outcome[(ast,k)])
-    if math.isinf(old_tm) or math.isinf(new_tm) or old_tm < 1e-9 or new_tm < 1e-9: continue
-    try:
-      lin = Kernel(eval(ast))
-    except Exception:
-      continue
-    lin.apply_opts(k[:-1])
-    act = k[-1]
-    log_ratio = math.log(old_tm/new_tm)
-    #print(f"ratio: {old_tm/new_tm:6.2f}x (log {log_ratio:5.2f}) from {str(act):50s} on {lin.colored_shape()}")
-    S.append(lin_to_feats(lin, use_sts=True))
-    A.append(actions.index(act))
-    V.append([log_ratio])  # NOTE: i have written the bug many times with this having the wrong dim
-
-  S, A, V = np.array(S), np.array(A), np.array(V, dtype=np.float32)
-  X = np.zeros((S.shape[0], S.shape[1]+len(actions)), dtype=np.float32)
-  X[:, :S.shape[1]] = S
-  X[range(S.shape[0]), S.shape[1]+A] = 1.0
-  return X, V
-
-def log_likelihood(x:Tensor, mu:Tensor, log_sigma:Tensor):
-  #print(x.shape, mu.shape, log_sigma.shape)
-  #return (x-mu).abs() * (-log_sigma).exp() + log_sigma
-  return (x-mu).square() * (-2*log_sigma).exp() / 2 + log_sigma
-
-if __name__ == "__main__":
-  if getenv("REGEN"):
-    X,V = dataset_from_cache(sys.argv[1] if len(sys.argv) > 1 else "/tmp/tinygrad_cache")
-    safe_save({"X": Tensor(X), "V": Tensor(V)}, "/tmp/dataset")
-  else:
-    ld = safe_load("/tmp/dataset")
-    X,V = ld['X'].numpy(), ld['V'].numpy()
-
-  print(X.shape, V.shape)
-  order = list(range(X.shape[0]))
-  random.shuffle(order)
-  X, V = X[order], V[order]
-
-  ratio = -512
-  X_test, V_test = Tensor(X[ratio:]), Tensor(V[ratio:])
-  X,V = X[:ratio], V[:ratio]
-  print(X.shape, V.shape)
-
-  #print(X[0], V[0])
-  #print(X[-1], V[-1])
-  print(X.shape)
-
-  net = ValueNet(X.shape[1], 2)
-  optim = Adam(get_parameters(net))
-
-  def get_minibatch(X,Y,bs):
-    xs, ys = [], []
-    #random.seed(1337)
-    for _ in range(bs):
-      sel = random.randint(0, len(X)-1)
-      xs.append(X[sel])
-      ys.append(Y[sel])
-    return Tensor(xs), Tensor(ys)
-
-  Tensor.training = True
-  losses = []
-  test_losses = []
-  test_loss = float('inf')
-  for i in (t:=trange(2000)):
-    x,y = get_minibatch(X,V,bs=256)
-    out = net(x)
-    #loss = (out-y).square().mean()
-    loss = log_likelihood(y, out[:, 0:1], out[:, 1:2]).mean()
-    optim.zero_grad()
-    loss.backward()
-    optim.step()
-    t.set_description(f"loss {loss.numpy():7.2f}, test loss {test_loss:7.2f}")
-    losses.append(loss.numpy().item())
-    test_losses.append(test_loss)
-    if i % 10: test_loss = (net(X_test)[:, 0:1]-V_test).square().mean().numpy().item()
-
-  safe_save(get_state_dict(net), "/tmp/qnet.safetensors")
-
-  import matplotlib.pyplot as plt
-  plt.plot(losses[20:])
-  plt.plot(test_losses[20:])
-  plt.show()

extra/optimization/helpers.py

@@ -1,124 +0,0 @@
-# stuff needed to unpack a kernel
-from tinygrad import Variable
-from tinygrad.codegen.opt import Opt, OptOps
-from tinygrad.uop.ops import UOp, Ops, KernelInfo
-from tinygrad.dtype import dtypes, PtrDType
-from tinygrad.shape.shapetracker import ShapeTracker
-from tinygrad.shape.view import View
-from tinygrad.helpers import getenv
-from tinygrad.engine.realize import get_program
-inf, nan = float('inf'), float('nan')
-UOps = Ops
-
-# kernel unpacker
-from tinygrad.codegen.opt.kernel import Kernel
-def ast_str_to_ast(ast_str:str) -> UOp: return eval(ast_str)
-def ast_str_to_lin(ast_str:str, opts=None): return Kernel(ast_str_to_ast(ast_str), opts=opts)
-def kern_str_to_lin(kern_str:str, opts=None):
-  (ast, applied_opts,) = eval(kern_str)
-  k = Kernel(ast, opts=opts)
-  k.apply_opts(applied_opts)
-  return k
-
-# load worlds, a dataset of about 12k kernels
-import gzip
-from pathlib import Path
-import random
-from tinygrad.helpers import dedup, DEBUG
-def load_worlds(filter_reduce=True, filter_noimage=True, filter_novariable=True):
-  fn = Path(__file__).parent.parent / "datasets/sops.gz"
-  ast_strs = dedup(gzip.open(fn).read().decode('utf-8').strip().split("\n"))
-  assert len(ast_strs) >= getenv("MIN_ASTS", 1000), f"dataset size = {len(ast_strs)} is too small"
-  if DEBUG >= 1: print(f"loaded {len(ast_strs)=} before filters")
-  if filter_reduce: ast_strs = [x for x in ast_strs if "REDUCE_AXIS" in x]
-  if filter_noimage: ast_strs = [x for x in ast_strs if "dtypes.image" not in x]
-  if filter_novariable: ast_strs = [x for x in ast_strs if "DEFINE_VAR" not in x]
-  if DEBUG >= 1: print(f"loaded {len(ast_strs)=} after filters {filter_reduce=}, {filter_noimage=}, {filter_novariable=}")
-  random.seed(1337)
-  random.shuffle(ast_strs)
-  return ast_strs
-
-def assert_same_lin(l1, l2):
-  assert l1.colored_shape() == l2.colored_shape()
-  assert all(x==y for x,y in zip(l1.sts, l2.sts))
-
-# get features
-import math
-
-MAX_DIMS = 16
-MAX_BUFS = 9
-def lin_to_feats(lin:Kernel, use_sts=True):
-  assert lin.shape_len < MAX_DIMS, "too many dims"
-
-  all_colors = ["blue", "cyan", "white", "green", "red", "magenta", "yellow"]
-  lc = [all_colors.index(x) for x in lin.colors()]
-
-  ret = []
-  # before, some generic linearizer stuff
-  ret.append(lin.upcasted)
-  ret.append(lin.local_dims)
-
-  # first, the full shape, including the colors
-  for s,os,c in zip(lin.full_shape,lin.output_shape,lc):
-    if isinstance(s, UOp):
-      ret.append(False)
-      ret += [0]*9
-    else:
-      ret.append(True)
-      ret.append(math.log2(s))
-      ret.append(min(33, s))
-      ret.append(math.log2(os))
-      ret.append(min(33, os))
-      ret.append(s%2 == 0)
-      ret.append(s%3 == 0)
-      ret.append(s%4 == 0)
-      ret.append(s%8 == 0)
-      ret.append(s%16 == 0)
-    cc = [0]*7
-    cc[c] = 1
-    ret += cc
-  ret += [0] * (17*(MAX_DIMS-len(lin.full_shape)))
-  ret = [float(x) for x in ret]
-
-  if use_sts:
-    my_sts = dedup([(x.shape == lin.full_shape, x.is_expanded(), any(v.mask is not None for v in x.views), len(x.views)) for x in lin.sts])
-    assert len(my_sts) < MAX_BUFS
-    sts_len = 3 + 5*MAX_DIMS
-    for s in my_sts:
-      ret.append(s[0])  # reduce
-      ret.append(s[2])  # has mask
-      ret.append(s[3])  # len views
-      for d in s[1]:
-        ret.append(d is None)
-        ret.append(d == 0)
-        ret.append(d == 1)
-        ret.append(min(33, d) if d is not None else -1)
-        if d is not None and d >= 1: ret.append(math.log2(d))
-        else: ret.append(-1)
-      ret += [0] * (5*(MAX_DIMS - len(s[1])))
-    ret += [0] * (sts_len*(MAX_BUFS - len(my_sts)))
-    assert len(ret) == 1021, f"wrong len {len(ret)}"
-  else:
-    assert len(ret) == 274, f"wrong len {len(ret)}"
-  return ret
-
-from tinygrad.device import Device, Buffer
-from tinygrad.codegen.opt.search import _ensure_buffer_alloc, _time_program
-from tinygrad.helpers import to_function_name, CACHELEVEL, diskcache_get, diskcache_put
-
-def time_linearizer(lin:Kernel, rawbufs:list[Buffer], allow_test_size=True, max_global_size=65536, cnt=3, disable_cache=False, clear_l2=False) -> float:  # noqa: E501
-  key = {"ast": lin.ast.key, "opts": str(lin.applied_opts), "allow_test_size": allow_test_size,
-         "max_global_size": max_global_size, "clear_l2": clear_l2, "device": lin.opts.device, "suffix": lin.opts.suffix}
-  if not disable_cache and CACHELEVEL >= 2 and (val:=diskcache_get("time_linearizer", key)) is not None: return min(val)
-
-  dev = Device[lin.opts.device]
-  assert dev.compiler is not None
-
-  rawbufs = _ensure_buffer_alloc(rawbufs)
-  var_vals: dict[str, int] = {k.expr:int(k.vmax+k.vmin)//2 for k in lin.ast.variables()}
-  p = get_program(lin.get_optimized_ast(), lin.opts)
-  tms = _time_program(p, dev.compiler.compile(p.src), var_vals, rawbufs,
-                      max_global_size=max_global_size if allow_test_size else None, clear_l2=clear_l2, cnt=cnt, name=to_function_name(lin.name))
-
-  if CACHELEVEL >= 2: diskcache_put("time_linearizer", key, tms)
-  return min(tms)

extra/optimization/pretrain_valuenet.py

@@ -1,88 +0,0 @@
-from tinygrad.codegen.opt.kernel import Kernel
-from tqdm import tqdm, trange
-import math
-import random
-from tinygrad.tensor import Tensor
-from tinygrad.nn import Linear
-from tinygrad.nn.optim import Adam
-from tinygrad.nn.state import get_parameters, get_state_dict, safe_save, safe_load, load_state_dict
-
-# stuff needed to unpack a kernel
-from tinygrad.uop.ops import LazyOp, TernaryOps, BinaryOps, UnaryOps, ReduceOps, BufferOps, MemBuffer, ConstBuffer
-from tinygrad.dtype import dtypes
-from tinygrad.shape.shapetracker import ShapeTracker
-from tinygrad.shape.view import View
-from tinygrad.uop.ops import Variable
-inf, nan = float('inf'), float('nan')
-from tinygrad.codegen.opt.kernel import Opt, OptOps
-
-from extra.optimization.helpers import lin_to_feats, MAX_DIMS
-
-# NOTE: this is not real value of the state, it's just a prediction of the runtime
-INNER = 512
-class ValueNet:
-  def __init__(self, feats=240, out=1):
-    self.l1 = Linear(feats,INNER)
-    self.l2 = Linear(INNER,INNER)
-    self.l3 = Linear(INNER,INNER)
-    self.l4 = Linear(INNER,out)
-  def __call__(self, x):
-    x = self.l1(x).relu()
-    x = self.l2(x).relu()
-    x = self.l3(x).relu().dropout(0.8)
-    return self.l4(x)
-
-if __name__ == "__main__":
-  net = ValueNet()
-  optim = Adam(get_parameters(net))
-
-  TEST_SIZE = 256
-
-  dset = open("/tmp/logtm").read().strip().split("\n")
-  random.seed(1337)
-  random.shuffle(dset)
-
-  X,Y = [], []
-  for i,x in enumerate(tqdm(dset)):
-    ast, opts, tms = eval(x)
-    lin = Kernel(ast)
-    for o in opts: lin.apply_opt(o)
-    if lin.shape_len >= MAX_DIMS: continue
-    if min(tms) == float('inf'): continue
-    X.append(lin_to_feats(lin))
-    Y.append([math.log(min(tms))])
-  print(f"got {len(X)} samples")
-
-  X_test,Y_test = Tensor(X[-TEST_SIZE:]), Tensor(Y[-TEST_SIZE:])
-  X,Y = X[:-TEST_SIZE], Y[:-TEST_SIZE]
-
-  def get_minibatch(X,Y,bs):
-    xs, ys = [], []
-    for _ in range(bs):
-      sel = random.randint(0, len(X)-1)
-      xs.append(X[sel])
-      ys.append(Y[sel])
-    return Tensor(xs), Tensor(ys)
-
-  Tensor.training = True
-  losses = []
-  test_losses = []
-  test_loss = float('inf')
-  for i in (t:=trange(2000)):
-    x,y = get_minibatch(X,Y,bs=256)
-    out = net(x)
-    loss = (out-y).square().mean()
-    optim.zero_grad()
-    loss.backward()
-    optim.step()
-    t.set_description(f"loss {loss.numpy():7.2f}, test loss {test_loss:7.2f}")
-    losses.append(loss.numpy().item())
-    test_losses.append(test_loss)
-    if i % 10: test_loss = (net(X_test)-Y_test).square().mean().numpy().item()
-
-  safe_save(get_state_dict(net), "/tmp/valuenet.safetensors")
-
-  import matplotlib.pyplot as plt
-  plt.plot(losses[200:])
-  plt.plot(test_losses[200:])
-  plt.show()

extra/remu/src/lib.rs

@@ -1,4 +1,5 @@
-use crate::work_group::WorkGroup;
+use crate::state::StateSnapshot;
+use crate::work_group::{WaveContext, WorkGroup};
 use std::os::raw::c_char;
 use std::slice;
 mod helpers;
@@ -30,3 +31,47 @@ pub extern "C" fn run_asm(lib: *const c_char, lib_sz: u32, gx: u32, gy: u32, gz:
     }
     0
 }
+
+// FFI functions for single-stepping comparison tests
+
+#[no_mangle]
+pub extern "C" fn wave_create(lib: *const c_char, lib_sz: u32, n_lanes: u32) -> *mut WaveContext {
+    if lib.is_null() || (lib_sz % 4) != 0 { return std::ptr::null_mut(); }
+    let kernel = unsafe { slice::from_raw_parts(lib as *const u32, (lib_sz / 4) as usize).to_vec() };
+    Box::into_raw(Box::new(WaveContext::new(kernel, n_lanes as usize)))
+}
+
+#[no_mangle]
+pub extern "C" fn wave_step(ctx: *mut WaveContext) -> i32 {
+    if ctx.is_null() { return -99; }
+    unsafe { (*ctx).step() }
+}
+
+#[no_mangle]
+pub extern "C" fn wave_get_snapshot(ctx: *const WaveContext, out: *mut StateSnapshot) {
+    if ctx.is_null() || out.is_null() { return; }
+    unsafe { *out = (*ctx).get_snapshot(); }
+}
+
+#[no_mangle]
+pub extern "C" fn wave_set_sgpr(ctx: *mut WaveContext, idx: u32, val: u32) {
+    if ctx.is_null() || idx >= 128 { return; }
+    unsafe { (*ctx).scalar_reg[idx as usize] = val; }
+}
+
+#[no_mangle]
+pub extern "C" fn wave_set_vgpr(ctx: *mut WaveContext, lane: u32, idx: u32, val: u32) {
+    if ctx.is_null() || lane >= 32 || idx >= 256 { return; }
+    unsafe { (*ctx).vec_reg.get_lane_mut(lane as usize)[idx as usize] = val; }
+}
+
+#[no_mangle]
+pub extern "C" fn wave_init_lds(ctx: *mut WaveContext, size: u32) {
+    if ctx.is_null() { return; }
+    unsafe { (*ctx).lds.data.resize(size as usize, 0); }
+}
+
+#[no_mangle]
+pub extern "C" fn wave_free(ctx: *mut WaveContext) {
+    if !ctx.is_null() { unsafe { drop(Box::from_raw(ctx)); } }
+}

extra/remu/src/state.rs

@@ -96,6 +96,24 @@ impl WaveValue {
     }
 }
 
+/// C-compatible state snapshot for FFI - used for comparing emulator states
+#[repr(C)]
+#[derive(Clone, Debug)]
+pub struct StateSnapshot {
+    pub pc: u32,
+    pub scc: u32,
+    pub vcc: u32,
+    pub exec_mask: u32,
+    pub sgpr: [u32; 128],
+    pub vgpr: [[u32; 256]; 32],
+}
+
+impl StateSnapshot {
+    pub fn new() -> Self {
+        Self { pc: 0, scc: 0, vcc: 0, exec_mask: 0, sgpr: [0; 128], vgpr: [[0; 256]; 32] }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct VecDataStore {
     pub data: Vec<u8>,

extra/remu/src/work_group.rs

@@ -1,5 +1,5 @@
 use crate::helpers::{colored, DEBUG};
-use crate::state::{Register, VecDataStore, WaveValue, VGPR};
+use crate::state::{Register, StateSnapshot, VecDataStore, WaveValue, VGPR};
 use crate::thread::{Thread, END_PRG, SGPR_COUNT};
 use std::collections::HashMap;
 
@@ -28,6 +28,96 @@ struct WaveState {
 
 const SYNCS: [u32; 4] = [0xBF89FC07, 0xBC7C0000, 0xBF890007, 0xbFB60003];
 const S_BARRIER: u32 = 0xBFBD0000;
+
+/// Context for single-stepping through a wave - holds all mutable state
+pub struct WaveContext {
+    pub kernel: Vec<u32>,
+    pub scalar_reg: [u32; SGPR_COUNT],
+    pub scc: u32,
+    pub pc: usize,
+    pub vec_reg: VGPR,
+    pub vcc: WaveValue,
+    pub exec: WaveValue,
+    pub lds: VecDataStore,
+    pub sds: HashMap<usize, VecDataStore>,
+    pub n_lanes: usize,
+}
+
+impl WaveContext {
+    pub fn new(kernel: Vec<u32>, n_lanes: usize) -> Self {
+        let active = (!0u32).wrapping_shr(32 - (n_lanes as u32));
+        Self {
+            kernel,
+            scalar_reg: [0; SGPR_COUNT],
+            scc: 0,
+            pc: 0,
+            vec_reg: VGPR::new(),
+            vcc: WaveValue::new(0, n_lanes),
+            exec: WaveValue::new(active, n_lanes),
+            lds: VecDataStore::new(),
+            sds: (0..=31).map(|i| (i, VecDataStore::new())).collect(),
+            n_lanes,
+        }
+    }
+
+    /// Execute a single instruction. Returns: 0=continue, -1=endpgm, -2=barrier, 1=done (pc past program), negative=error
+    pub fn step(&mut self) -> i32 {
+        if self.pc >= self.kernel.len() { return 1; }
+        if self.kernel[self.pc] == END_PRG { return -1; }
+        if self.kernel[self.pc] == S_BARRIER { self.pc += 1; return -2; }
+        // Skip sync/nop instructions
+        if SYNCS.contains(&self.kernel[self.pc]) || self.kernel[self.pc] >> 20 == 0xbf8 || self.kernel[self.pc] == 0x7E000000 {
+            self.pc += 1;
+            return 0;
+        }
+
+        let mut sgpr_co = None;
+        for lane_id in 0..self.n_lanes {
+            self.vec_reg.default_lane = Some(lane_id);
+            self.vcc.default_lane = Some(lane_id);
+            self.exec.default_lane = Some(lane_id);
+            let mut thread = Thread {
+                scalar_reg: &mut self.scalar_reg,
+                scc: &mut self.scc,
+                vec_reg: &mut self.vec_reg,
+                vcc: &mut self.vcc,
+                exec: &mut self.exec,
+                lds: &mut self.lds,
+                sds: &mut self.sds.get_mut(&lane_id).unwrap(),
+                pc_offset: 0,
+                stream: self.kernel[self.pc..].to_vec(),
+                scalar: false,
+                simm: None,
+                warp_size: self.n_lanes,
+                sgpr_co: &mut sgpr_co,
+            };
+            if let Err(e) = thread.interpret() { return e; }
+            if thread.scalar {
+                self.pc = ((self.pc as isize) + 1 + (thread.pc_offset as isize)) as usize;
+                break;
+            }
+            if lane_id == self.n_lanes - 1 {
+                self.pc = ((self.pc as isize) + 1 + (thread.pc_offset as isize)) as usize;
+            }
+        }
+        if self.vcc.mutations.is_some() { self.vcc.apply_muts(); self.vcc.mutations = None; }
+        if self.exec.mutations.is_some() { self.exec.apply_muts(); self.exec.mutations = None; }
+        if let Some((idx, mut wv)) = sgpr_co.take() { wv.apply_muts(); self.scalar_reg[idx] = wv.value; }
+        0
+    }
+
+    pub fn get_snapshot(&self) -> StateSnapshot {
+        let mut snap = StateSnapshot::new();
+        snap.pc = self.pc as u32;
+        snap.scc = self.scc;
+        snap.vcc = self.vcc.value;
+        snap.exec_mask = self.exec.value;
+        snap.sgpr = self.scalar_reg;
+        for lane in 0..32 { snap.vgpr[lane] = self.vec_reg.get_lane(lane); }
+        snap
+    }
+}
+
 impl<'a> WorkGroup<'a> {
     pub fn new(dispatch_dim: u32, id: [u32; 3], launch_bounds: [u32; 3], kernel: &'a Vec<u32>, kernel_args: *const u64) -> Self {
         Self { dispatch_dim, id, kernel, launch_bounds, kernel_args, lds: VecDataStore::new(), wave_state: HashMap::new() }

extra/remu/test/hwtest.py

@@ -1,32 +1,37 @@
+# ruff: noqa: F405, F403
+# allow define from star imports
+
 import numpy as np
 import unittest
-import subprocess, struct, math
+import subprocess, struct, math, textwrap
 from tinygrad import Tensor, dtypes, Device, UOp
+from tinygrad.uop.ops import Ops
 from tinygrad.helpers import getenv
 from tinygrad.runtime.support.compiler_amd import amdgpu_disassemble
 from tinygrad.renderer import ProgramSpec
 from tinygrad.engine.realize import CompiledRunner
 
-def get_output(asm:str, n_threads:int=1):
-  input_asm = "\n".join([ln if ln.strip().startswith('asm volatile') else f'asm volatile("{ln.strip().lstrip()}" : "+v"(a), "+v"(b));'
-                         for ln in asm.strip().splitlines() if ln.strip()])
-  src = f"""
-  typedef long unsigned int size_t;
-  extern "C" __attribute__((device, const)) size_t __ockl_get_local_id(unsigned int);
-  extern "C" __attribute__((global)) void __attribute__((amdgpu_flat_work_group_size(1, {n_threads}))) test(unsigned int* data0_1) {{
-    int l = __ockl_get_local_id(0);
-    unsigned a = 0, b = 0, c = 0;
-    {input_asm}
-    unsigned res;
-    asm volatile("v_mov_b32 %0, %1" : "=v"(res) : "v"(a));
-    *(data0_1+l) = res;
-  }}"""
-  t = Tensor.zeros(n_threads, dtype=dtypes.uint32).contiguous().realize()
-  prg = ProgramSpec("test", src, Device.DEFAULT, UOp.sink(t), global_size=[1, 1, 1], local_size=[n_threads, 1, 1])
+from extra.assembly.amd.autogen.rdna3 import *
+from extra.assembly.amd.asm import waitcnt
+from test.testextra.test_cfg_viz import template
+
+def get_output(asm:list, n_threads:int=1, vdst:VGPR=v[1]):
+  out = Tensor([0]*n_threads, dtype=dtypes.uint32).realize()
+  src = "\n".join(inst.disasm() for inst in [
+    s_load_b64(s[0:1], s[0:1], NULL),
+    *asm,
+    v_lshlrev_b32_e32(v[0], 2, v[0]),
+    s_waitcnt(simm16=waitcnt(lgkmcnt=0)),
+    #global_store_b32(v[0], v[1], s[0:1]),
+    global_store_b32(addr=v[0], data=vdst, saddr=s[0:1]),
+    s_endpgm()
+  ])
+  prg = ProgramSpec("test", template.replace("fn_name", "test").replace("INSTRUCTION", textwrap.dedent(src)), Device.DEFAULT, UOp(Ops.SINK),
+                    global_size=[1, 1, 1], local_size=[n_threads, 1, 1], globals=[0])
   car = CompiledRunner(prg)
   if getenv("PRINT_ASM"): amdgpu_disassemble(car.lib)
-  car([t.uop.buffer], {}, wait=True)
-  return t.numpy()
+  car([out.uop.buffer], {}, wait=True)
+  return out.tolist()
 
 def f16_to_bits(x:float) -> int: return struct.unpack('<H', struct.pack('<e', x))[0]
 def f32_from_bits(x:int) -> float: return struct.unpack('<f', struct.pack('<I', x))[0]
@@ -37,83 +42,89 @@ class TestHW(unittest.TestCase):
   def setUp(self):
     if getenv("MOCKGPU"): subprocess.run(["cargo", "build", "--release", "--manifest-path", "./extra/remu/Cargo.toml"], check=True)
 
-  def test_simple(self):
-    out = get_output("""
-    v_mov_b32_e32 %1 42
-    v_mov_b32_e32 %2 %1
-    """)[0]
-    np.testing.assert_equal(out, 42)
+  def test_simple_v_mov(self):
+    out = get_output([
+      v_mov_b32_e32(v[1], 2),
+    ])
+    self.assertEqual(out, [2])
+
+  def test_simple_s_mov(self):
+    out = get_output([
+      s_mov_b32(s[7], 0x7fffffff),
+      v_mov_b32_e32(v[1], s[7]),
+    ])
+    self.assertEqual(out, [0x7fffffff])
 
   def test_exec_mov(self):
-    out = get_output("""
-    v_mov_b32_e32 %1 42
-    s_mov_b32_e32 exec_lo 0b10
-    v_mov_b32_e32 %1 10
-    s_mov_b32_e32 exec_lo 0b11
-    v_mov_b32_e32 %2 %1
-    """, n_threads=2)
+    out = get_output([
+      v_mov_b32_e32(v[1], 42),
+      s_mov_b32(EXEC_LO, 0b10),
+      v_mov_b32_e32(v[1], 10),
+      s_mov_b32(EXEC_LO, 0b11),
+    ], n_threads=2)
     np.testing.assert_equal(out, [42, 10])
 
   def test_exec_cmp_vopc(self):
-    out = get_output("""
-    s_mov_b32 vcc_lo 0 // reset vcc
-    v_mov_b32_e32 %1 42
-    v_mov_b32_e32 %2 10
-    s_mov_b32_e32 exec_lo 0b01
-    v_cmp_ne_u32 %1 %2
-    s_mov_b32_e32 exec_lo 0b11
-    v_mov_b32_e32 %2 vcc_lo
-    """, n_threads=2)
-    np.testing.assert_equal(out, 0b01)
+    out = get_output([
+      s_mov_b32(VCC_LO, 0), # reset vcc
+      v_mov_b32_e32(v[1], 42),
+      v_mov_b32_e32(v[2], 10),
+      s_mov_b32(EXEC_LO, 0b01),
+      v_cmp_ne_u32_e32(v[1], v[2]),
+      s_mov_b32(EXEC_LO, 0b11),
+      v_mov_b32_e32(v[1], VCC_LO),
+    ], n_threads=2)[0]
+    np.testing.assert_equal(out, 1)
 
   def test_exec_cmpx_vop3(self):
-    out = get_output("""
-    s_mov_b32_e32 exec_lo 0b11
-    v_mov_b32_e32 %1 42
-    v_mov_b32_e32 %2 10
-    s_mov_b32_e32 exec_lo 0b01
-    v_cmpx_ne_u32 %1 %2
-    s_mov_b32_e32 s10 exec_lo
-    s_mov_b32_e32 exec_lo 0b11
-    v_mov_b32_e32 %2 s10
-    """, n_threads=2)[0]
+    out = get_output([
+      s_mov_b32(EXEC_LO, 0b11),
+      v_mov_b32_e32(v[1], 42),
+      v_mov_b32_e32(v[2], 10),
+      s_mov_b32(EXEC_LO, 0b01),
+      v_cmpx_ne_u32_e32(v[1], v[2]),
+      s_mov_b32(s[10], EXEC_LO),
+      s_mov_b32(EXEC_LO, 0b11),
+      v_mov_b32_e32(v[1], s[10]),
+    ], n_threads=2)[0]
     np.testing.assert_equal(out & 0b11, 0b01)
 
   def test_fmac_vop3_modifier(self):
-    init_state = f"""
-    asm volatile("v_mov_b32_e32 %1, {f16_to_bits(4.0)}" : "+v"(a));
-    asm volatile("v_mov_b32_e32 %1, {f16_to_bits(3.0)}" : "+v"(b));
-    asm volatile("v_mov_b32_e32 %1, {f16_to_bits(2.0)}" : "+v"(c));
-    """
-    mov = """asm volatile("v_mov_b32_e32 %1, %2" : "+v"(c), "+v"(a));"""
-    def fmac(a, b, c): return f"""asm volatile("v_fmac_f16_e64 {c}, {a}, {b}" : "+v"(c) : "v"(a), "v"(b));"""+"\n"+mov
-    self.assertEqual(get_output(init_state+"\n"+fmac("%1", "%2", "%3")), f16_to_bits(14.))
-    self.assertEqual(get_output(init_state+"\n"+fmac("%1", "-%2", "%3")), f16_to_bits(-10.))
-    self.assertEqual(get_output(init_state+"\n"+fmac("-%1", "-%2", "%3")), f16_to_bits(14.))
+    init_state = [
+      v_mov_b32_e32(a:=v[1], f16_to_bits(4.0)),
+      v_mov_b32_e32(b:=v[2], f16_to_bits(3.0)),
+      v_mov_b32_e32(c:=v[3], f16_to_bits(2.0)),
+    ]
+    def run_fmac(a, b): return get_output(init_state+[v_fmac_f16_e64(c, a, b)], vdst=c)[0]
+    self.assertEqual(run_fmac(a, b), f16_to_bits(14.0))
+    self.assertEqual(run_fmac(a, -b), f16_to_bits(-10.0))
+    self.assertEqual(run_fmac(-a, -b), f16_to_bits(14.0))
 
   def test_s_abs_i32(self):
-    def s_abs_i32(x, y, dst="s10", scc=0):
-      for reg,val in [(dst, y), ("scc", scc)]:
-        self.assertEqual(get_output(f"""
-        s_mov_b32_e32 {dst} {x}
-        s_abs_i32 {dst} {dst}
-        v_mov_b32_e32 %2 {reg}
-        """)[0], val)
-    s_abs_i32(0x00000001, 0x00000001, scc=1)
-    s_abs_i32(0x7fffffff, 0x7fffffff, scc=1)
-    s_abs_i32(0x80000000, 0x80000000, scc=1)
-    s_abs_i32(0x80000001, 0x7fffffff, scc=1)
-    s_abs_i32(0x80000002, 0x7ffffffe, scc=1)
-    s_abs_i32(0xffffffff, 0x00000001, scc=1)
-    s_abs_i32(0, 0, scc=0)
+    def check(x, y, dst=s[10], scc=0):
+      for reg,val in [(dst, y), (SCC, scc)]:
+        self.assertEqual(get_output([
+          s_mov_b32(dst, x),
+          s_abs_i32(dst, dst),
+          v_mov_b32_e32(v[1], reg)
+        ])[0], val)
+
+    check(0x00000001, 0x00000001, scc=1)
+    check(0x7fffffff, 0x7fffffff, scc=1)
+    check(0x80000000, 0x80000000, scc=1)
+    check(0x80000001, 0x7fffffff, scc=1)
+    check(0x80000002, 0x7ffffffe, scc=1)
+    check(0xffffffff, 0x00000001, scc=1)
+    check(0, 0, scc=0)
 
   def test_v_rcp_f32_neg_vop3(self):
     def v_neg_rcp_f32(x:float, y:float):
-      out = get_output(f"""
-      v_mov_b32_e32 %2 {f32_to_bits(x)}
-      v_rcp_f32_e64 %2, -%2
-      """)[0]
+      out = get_output([
+        v_mov_b32_e32(v[2], f32_to_bits(x)),
+        v_rcp_f32_e64(v[2], -v[2]),
+      ], vdst=v[2])[0]
       assert out == f32_to_bits(y), f"{f32_from_bits(out)} != {y} / {out} != {f32_to_bits(y)}"
+
     v_neg_rcp_f32(math.inf, -0.0)
     v_neg_rcp_f32(-math.inf, 0.0)
     v_neg_rcp_f32(0.0, -math.inf)
@@ -122,25 +133,28 @@ class TestHW(unittest.TestCase):
     v_neg_rcp_f32(2.0, -0.5)
 
   def test_v_cndmask_b32_neg(self):
-    def v_neg(x:int|float, y:float):
-      # always pick -v1
-      out = get_output(f"""
-      v_mov_b32_e32 %2 {f32_to_bits(x)}
-      s_mov_b32_e32 s10 1
-      v_cndmask_b32 %2, %2, -%2 s10
-      """)[0]
+    def v_neg(x:float, y:float):
+      out = get_output([
+        v_mov_b32_e32(v[1], f32_to_bits(x)),
+        s_mov_b32(s[10], 1),
+        v_cndmask_b32_e64(v[1], v[1], -v[1], s[10]),
+      ])[0]
       assert out == f32_to_bits(y), f"{f32_from_bits(out)} != {y} / {out} != {f32_to_bits(y)}"
+
     v_neg(-0.0, 0.0)
     v_neg(0.0, -0.0)
     v_neg(2.0, -2.0)
     v_neg(math.inf, -math.inf)
     v_neg(-math.inf, math.inf)
 
+  @unittest.skip("how does VOPD work in the dsl")
   def test_v_subrev_wrap(self):
-    out = get_output("""
-    v_dual_mov_b32 %1, 0xffffffff :: v_dual_mov_b32 %2, 0x0
-    v_subrev_co_u32 %2, vcc_lo, %2, %1
-    """)[0]
+    out = get_output([
+      #v_dual_mov_b32(v[1], 0xffffffff, v[2], 0x0),
+      #v_dual_mov_b32(vdstx=v[1], srcx=0xffffffff, vdsty=v[2], srcy=0x0),
+      #VOPD(opx=VOPDOp.V_DUAL_MOV_B32, opy=VOPDOp.V_DUAL_MOV_B32, vdstx=v[1], srcx=0xffffffff, vdsty=v[2], srcy=0x0),
+      v_subrev_co_u32(v[2], VCC_LO, v[2], v[1]),
+    ], vdst=v[2])[0]
     self.assertEqual(out, 0xffff_ffff)
 
 if __name__ == "__main__":

extra/sqtt/attempt_sqtt_parse.py

@@ -534,8 +534,8 @@ def parse_sqtt_print_packets(data: bytes, filter=DEFAULT_FILTER, verbose=True) -
 
 def parse(fn:str):
   with Timing(f"unpickle {fn}: "): dat = pickle.load(open(fn, "rb"))
-  if getenv("ROCM", 0):
-    with Timing(f"decode {fn}: "): ctx = decode(dat)
+  #if getenv("ROCM", 0):
+  #  with Timing(f"decode {fn}: "): ctx = decode(dat)
   dat_sqtt = [x for x in dat if isinstance(x, ProfileSQTTEvent)]
   print(f"got {len(dat_sqtt)} SQTT events in {fn}")
   return dat_sqtt

extra/sqtt/roc.py

@@ -1,33 +1,9 @@
-import ctypes, pathlib, argparse, pickle, re, functools, dataclasses, itertools, threading
+#!/usr/bin/env python3
+import ctypes, pathlib, argparse, pickle, dataclasses, threading
 from typing import Generator
 from tinygrad.helpers import temp, unwrap, DEBUG
-from tinygrad.device import ProfileEvent, ProfileDeviceEvent, ProfileProgramEvent
-from tinygrad.runtime.ops_amd import ProfileSQTTEvent, ProfilePMCEvent
-from tinygrad.runtime.autogen import llvm, rocprof
-from tinygrad.runtime.support.elf import elf_loader
-
-# to pass NULL to callbacks
-llvm.LLVMCreateDisasmCPUFeatures.argtypes = tuple(llvm.LLVMCreateDisasmCPUFeatures.argtypes[:5]) + (ctypes.c_void_p, ctypes.c_void_p)
-def llvm_disasm(arch:str, lib:bytes) -> dict[int, tuple[str, int]]:
-  llvm.LLVMInitializeAMDGPUTargetInfo()
-  llvm.LLVMInitializeAMDGPUTargetMC()
-  llvm.LLVMInitializeAMDGPUAsmParser()
-  llvm.LLVMInitializeAMDGPUDisassembler()
-  ctx = llvm.LLVMCreateDisasmCPUFeatures("amdgcn-amd-amdhsa".encode(), arch.encode(), "".encode(), None, 0, None, None)
-
-  image, sections, relocs = elf_loader(lib)
-  text = next((sh.header for sh in sections if sh.name == ".text"), None)
-  off, sz = unwrap(text).sh_addr, unwrap(text).sh_size
-
-  addr_table:dict[int, tuple[str, int]] = {}
-  out = ctypes.create_string_buffer(128)
-  cur_off = off
-  while cur_off < sz + off:
-    view = (ctypes.c_ubyte * ((sz + off) - cur_off)).from_buffer_copy(memoryview(image)[cur_off:])
-    instr_sz = llvm.LLVMDisasmInstruction(ctx, view, ctypes.c_uint64(len(view)), ctypes.c_uint64(0), out, ctypes.c_size_t(128))
-    addr_table[cur_off] = (out.value.decode("utf-8", "replace").strip(), instr_sz)
-    cur_off += instr_sz
-  return addr_table
+from tinygrad.runtime.ops_amd import ProfileSQTTEvent
+from tinygrad.runtime.autogen import rocprof
 
 @dataclasses.dataclass(frozen=True)
 class InstExec:
@@ -70,17 +46,11 @@ class OccEvent(WaveSlot):
 RunKey = tuple[str, int]
 
 class _ROCParseCtx:
-  def __init__(self, dev_evs:dict[str, ProfileDeviceEvent], sqtt_evs:list[ProfileSQTTEvent], prog_evs:list[ProfileProgramEvent]):
-    self.dev_evs, self.sqtt_evs, self.prog_evs = dev_evs, iter(sqtt_evs), prog_evs
-    self.disasms:dict[str, dict[int, tuple[str, int]]] = {}
+  def __init__(self, sqtt_evs:list[ProfileSQTTEvent], disasms:dict[str, dict[int, tuple[str, int]]]):
+    self.sqtt_evs, self.disasms = iter(sqtt_evs), disasms
     self.inst_execs:dict[RunKey, list[WaveExec]] = {}
     self.occ_events:dict[RunKey, list[OccEvent]] = {}
 
-    for prog in prog_evs:
-      arch = "gfx%d%x%x" % ((trgt:=unwrap(dev_evs[prog.device].props)['gfx_target_version']) // 10000, (trgt // 100) % 100, trgt % 100)
-      base = unwrap(prog.base)
-      self.disasms[prog.name] = asm = {base+addr:info for addr,info in llvm_disasm(arch, unwrap(prog.lib)).items()}
-
   def next_sqtt(self):
     x = next(self.sqtt_evs, None)
     self.active_run = (x.kern, x.exec_tag) if x is not None else None
@@ -103,16 +73,8 @@ class _ROCParseCtx:
     self.inst_execs.setdefault(unwrap(self.active_run), []).append(WaveExec(ev.wave_id, ev.cu, ev.simd, unwrap(self.active_se), ev.begin_time,
                                                                              ev.end_time, insts_blob))
 
-def decode(profile:list[ProfileEvent]) -> _ROCParseCtx:
-  dev_events:dict[str, ProfileDeviceEvent] = {}
-  sqtt_events:list[ProfileSQTTEvent] = []
-  prog_events:list[ProfileProgramEvent] = []
-  for e in profile:
-    if isinstance(e, ProfileDeviceEvent): dev_events[e.device] = e
-    if isinstance(e, ProfileSQTTEvent): sqtt_events.append(e)
-    if isinstance(e, ProfileProgramEvent) and e.device.startswith("AMD"): prog_events.append(e)
-
-  ROCParseCtx = _ROCParseCtx(dev_events, sqtt_events, prog_events)
+def decode(sqtt_evs:list[ProfileSQTTEvent], disasms:dict[str, dict[int, tuple[str, int]]]) -> _ROCParseCtx:
+  ROCParseCtx = _ROCParseCtx(sqtt_evs, disasms)
 
   @rocprof.rocprof_trace_decoder_se_data_callback_t
   def copy_cb(buf, buf_size, _):
@@ -153,26 +115,52 @@ def decode(profile:list[ProfileEvent]) -> _ROCParseCtx:
 
   def worker():
     try: rocprof.rocprof_trace_decoder_parse_data(copy_cb, trace_cb, isa_cb, None)
-    except AttributeError as e: raise RuntimeError("Failed to find rocprof-trace-decoder. Run sudo ./extra/sqtt/install_sqtt_decoder.py to install") from e
+    except AttributeError as e:
+      raise RuntimeError("Failed to find rocprof-trace-decoder. Run sudo ./extra/sqtt/install_sqtt_decoder.py to install") from e
   (t:=threading.Thread(target=worker, daemon=True)).start()
   t.join()
   return ROCParseCtx
 
-def print_pmc(events:list[ProfilePMCEvent]) -> None:
-  from tinygrad.viz.serve import unpack_pmc
+def print_data(data:dict) -> None:
   from tabulate import tabulate
-  for e in events:
-    print("**", e.kern)
-    data = unpack_pmc(e)
-    print(tabulate([r[:-1] for r in data["rows"]], headers=data["cols"], tablefmt="github"))
+  # plaintext
+  if "src" in data: print(data["src"])
+  # table format
+  elif "cols" in data:
+    print(tabulate([r[:len(data["cols"])] for r in data["rows"]], headers=data["cols"], tablefmt="github"))
+
+def main() -> None:
+  import tinygrad.viz.serve as viz
+  viz.ctxs = []
 
-if __name__ == "__main__":
   parser = argparse.ArgumentParser()
-  parser.add_argument('--profile', type=pathlib.Path, help='Path to profile', default=pathlib.Path(temp("profile.pkl", append_user=True)))
+  parser.add_argument('--profile', type=pathlib.Path, metavar="PATH", help='Path to profile (optional file, default: latest profile)',
+                      default=pathlib.Path(temp("profile.pkl", append_user=True)))
+  parser.add_argument('--kernel', type=str, default=None, metavar="NAME", help='Kernel to focus on (optional name, default: all kernels)')
+  parser.add_argument('-n', type=int, default=3, metavar="NUM", help='Max traces to print (optional number, default: 3 traces)')
   args = parser.parse_args()
 
   with args.profile.open("rb") as f: profile = pickle.load(f)
-  rctx = decode(profile)
-  print('SQTT:', rctx.inst_execs.keys())
 
-  print_pmc([ev for ev in profile if isinstance(ev, ProfilePMCEvent)])
+  viz.get_profile(profile)
+
+  # List all kernels
+  if args.kernel is None:
+    for c in viz.ctxs:
+      print(c["name"])
+      for s in c["steps"]: print("  "+s["name"])
+    return None
+
+  # Find kernel trace
+  trace = next((c for c in viz.ctxs if c["name"] == f"Exec {args.kernel}"), None)
+  if not trace: raise RuntimeError(f"no matching trace for {args.kernel}")
+  n = 0
+  for s in trace["steps"]:
+    print(s["name"])
+    data = viz.get_render(s["query"])
+    print_data(data)
+    n += 1
+    if n > args.n: break
+
+if __name__ == "__main__":
+  main()

extra/test_pyrender.py

@@ -1,20 +0,0 @@
-from extra.optimization.helpers import load_worlds, ast_str_to_ast
-from tinygrad.helpers import tqdm
-from tinygrad.uop.ops import pyrender, UOp, Ops
-from tinygrad import dtypes
-from tinygrad.shape.shapetracker import ShapeTracker, View
-inf, nan = float('inf'), float('nan')
-
-if __name__ == "__main__":
-  ast_strs = load_worlds()
-  for i, ast_str in enumerate(tqdm(ast_strs)):
-    good_ast = ast_str_to_ast(ast_str)
-    code = '\n'.join(pyrender(good_ast))
-    print("\n***************\n\n"+code)
-    exec(code)
-    if str(good_ast) != str(ast):
-      print(code)
-      print("MISMATCH")
-      print(good_ast)
-      print(ast)
-      break

opencode.json

@@ -1 +1 @@
-{"$schema": "https://opencode.ai/config.json", "formatter": false}
+{"$schema": "https://opencode.ai/config.json", "formatter": false, "lsp": false}

test/device/test_hcq.py

@@ -76,7 +76,7 @@ class TestHCQ(unittest.TestCase):
         TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
         TestHCQ.d0.timeline_value += 1
 
-  @unittest.skipIf(MOCKGPU or Device.DEFAULT in {"CPU"}, "Can't handle async update on MOCKGPU for now")
+  @unittest.skipIf(Device.DEFAULT in {"CPU"}, "Can't handle async update on CPU device")
   def test_wait_late_set(self):
     for queue_type in [TestHCQ.d0.hw_compute_queue_t, TestHCQ.d0.hw_copy_queue_t]:
       if queue_type is None: continue

test/external/external_benchmark_op_conv.py

@@ -1,10 +1,9 @@
 # ruff: noqa: E501 E712 F401
+from dataclasses import replace
 from tinygrad import dtypes, Device
 from tinygrad.uop.ops import UOp, AxisType, Ops, KernelInfo
-from tinygrad.codegen import full_rewrite
 from tinygrad.codegen.opt import Opt, OptOps # pylint: disable=unused-import
-from tinygrad.renderer import ProgramSpec
-from tinygrad.engine.realize import CompiledRunner
+from tinygrad.engine.realize import CompiledRunner, get_program
 from tinygrad.helpers import dedup, getenv
 from tinygrad.device import Buffer
 from tinygrad.dtype import ImageDType, Invalid
@@ -86,16 +85,11 @@ def dm_conv_172():
 
 ast = {143: vision_conv_143, 153: vision_conv_153, 172: dm_conv_172}[getenv("NUM", 143)]()
 
-compiler = Device.default.compiler
 renderer = Device.default.renderer
 allocator = Device.default.allocator
 
-uops = full_rewrite(ast, renderer)
-src = renderer.render(uops)
-
-lib = compiler.compile(src)
-ps = ProgramSpec("conv", src, Device.DEFAULT, ast, uops)
-cr = CompiledRunner(ps, precompiled=lib)
+ps = get_program(ast, renderer)
+cr = CompiledRunner(replace(ps, device=Device.DEFAULT))
 
 gs = sorted(dedup([u for u in ast.toposort() if u.op is Ops.DEFINE_GLOBAL]), key=lambda u: u.arg)
 # print(len(gs))

test/external/external_hip_compiler_bug.py

@@ -1,225 +0,0 @@
-# [<buf device:HIP size:1605632 dtype:dtypes.float>, <buf device:HIP size:301506 dtype:dtypes.float>, <buf device:HIP size:9408 dtype:dtypes.float>]
-from tinygrad import Device, dtypes
-from tinygrad.device import Buffer, CompiledRunner
-
-import ctypes
-import gpuctypes.hip as hip
-from tinygrad.helpers import to_char_p_p, init_c_var
-def get_bytes(arg, get_sz, get_str, check) -> bytes: return (sz := init_c_var(ctypes.c_size_t(), lambda x: check(get_sz(arg, ctypes.byref(x)))), ctypes.string_at(init_c_var(ctypes.create_string_buffer(sz.value), lambda x: check(get_str(arg, x))), size=sz.value))[1]  # noqa: E501
-def check(status):
-  if status != 0: raise RuntimeError(f"HIP Error {status}, {ctypes.string_at(hip.hipGetErrorString(status)).decode()}")
-def compile_hip(prg:str, arch="gfx1100") -> bytes:
-  check(hip.hiprtcCreateProgram(ctypes.byref(prog := hip.hiprtcProgram()), prg.encode(), "<null>".encode(), 0, None, None))
-  compile_options = [f'--offload-arch={arch}', '-I/opt/rocm/include']
-  status = hip.hiprtcCompileProgram(prog, len(compile_options), to_char_p_p([o.encode() for o in compile_options]))
-  if status != 0: raise RuntimeError(f"compile failed: {get_bytes(prog, hip.hiprtcGetProgramLogSize, hip.hiprtcGetProgramLog, check).decode()}")
-  return get_bytes(prog, hip.hiprtcGetCodeSize, hip.hiprtcGetCode, check)
-
-prefix = """
-typedef long unsigned int size_t;
-extern "C" __attribute__((device)) __attribute__((const)) size_t __ockl_get_local_id(unsigned int);
-extern "C" __attribute__((device)) __attribute__((const)) size_t __ockl_get_group_id(unsigned int);
-extern "C" __attribute__((device)) __attribute__((const)) size_t __ockl_get_local_size(unsigned int);
-typedef float float2 __attribute__((ext_vector_type(2)));
-static inline __attribute__((device)) float2 make_float2(float x, float y) { return {x, y}; }
-"""
-
-code = """
-extern "C" __attribute__((global))void r_2_8_7_7_4_8_3_7_7_4_4_2_2(float* data0, const float* data1, const float* data2) {
-  int gidx0 = __ockl_get_group_id(2); /* 2 */
-  int gidx1 = __ockl_get_group_id(1); /* 8 */
-  int gidx2 = __ockl_get_group_id(0); /* 49 */
-  int lidx4 = __ockl_get_local_id(1); /* 4 */
-  int lidx5 = __ockl_get_local_id(0); /* 8 */
-  float2 acc0 = make_float2(0.0f,0.0f);
-  float2 acc1 = make_float2(0.0f,0.0f);
-  float2 acc2 = make_float2(0.0f,0.0f);
-  float2 acc3 = make_float2(0.0f,0.0f);
-  float2 acc4 = make_float2(0.0f,0.0f);
-  float2 acc5 = make_float2(0.0f,0.0f);
-  float2 acc6 = make_float2(0.0f,0.0f);
-  float2 acc7 = make_float2(0.0f,0.0f);
-  float2 acc8 = make_float2(0.0f,0.0f);
-  float2 acc9 = make_float2(0.0f,0.0f);
-  float2 acc10 = make_float2(0.0f,0.0f);
-  float2 acc11 = make_float2(0.0f,0.0f);
-  float2 acc12 = make_float2(0.0f,0.0f);
-  float2 acc13 = make_float2(0.0f,0.0f);
-  float2 acc14 = make_float2(0.0f,0.0f);
-  float2 acc15 = make_float2(0.0f,0.0f);
-  float2 acc16 = make_float2(0.0f,0.0f);
-  float2 acc17 = make_float2(0.0f,0.0f);
-  float2 acc18 = make_float2(0.0f,0.0f);
-  float2 acc19 = make_float2(0.0f,0.0f);
-  float2 acc20 = make_float2(0.0f,0.0f);
-  float2 acc21 = make_float2(0.0f,0.0f);
-  float2 acc22 = make_float2(0.0f,0.0f);
-  float2 acc23 = make_float2(0.0f,0.0f);
-  float2 acc24 = make_float2(0.0f,0.0f);
-  float2 acc25 = make_float2(0.0f,0.0f);
-  float2 acc26 = make_float2(0.0f,0.0f);
-  float2 acc27 = make_float2(0.0f,0.0f);
-  float2 acc28 = make_float2(0.0f,0.0f);
-  float2 acc29 = make_float2(0.0f,0.0f);
-  float2 acc30 = make_float2(0.0f,0.0f);
-  float2 acc31 = make_float2(0.0f,0.0f);
-  int alu0 = (gidx2/7);
-  int alu1 = (gidx2%7);
-  int alu2 = (alu1*32);
-  int alu3 = (lidx5*4);
-  int alu4 = ((gidx0*802816)+(gidx1*100352)+(alu0*1792)+(alu1*16)+(lidx4*448)+(lidx5*2));
-  for (int ridx0 = 0; ridx0 < 3; ridx0++) {
-    for (int ridx1 = 0; ridx1 < 7; ridx1++) {
-      int alu5 = ((alu0*(-32))+(lidx4*(-8))+(ridx1*(-1)));
-      bool alu6 = (alu5<(-2));
-      bool alu7 = (alu5<0);
-      bool alu8 = (((alu0*32)+(lidx4*8)+ridx1)<221);
-      for (int ridx2 = 0; ridx2 < 7; ridx2++) {
-        int alu9 = ((gidx0*150528)+(ridx0*50176)+(alu0*7168)+(lidx4*1792)+(ridx1*224)+alu2+alu3+ridx2);
-        int alu10 = ((alu1*(-32))+(lidx5*(-4))+(ridx2*(-1)));
-        bool alu11 = (alu10<(-2));
-        float val0 = 0.0f;
-        if ((alu6*alu11)) { val0 = data1[alu9+(-675)]; }
-        float val1 = 0.0f;
-        if ((alu7*alu11)) { val1 = data1[alu9+(-227)]; }
-        float val2 = 0.0f;
-        if (alu11) { val2 = data1[alu9+221]; }
-        float val3 = 0.0f;
-        if ((alu8*alu11)) { val3 = data1[alu9+669]; }
-        bool alu12 = (alu10<0);
-        bool alu13 = ((alu2+alu3+ridx2)<225);
-        float val4 = 0.0f;
-        if ((alu6*alu12*alu13)) { val4 = data1[alu9+(-673)]; }
-        float val5 = 0.0f;
-        if ((alu7*alu12*alu13)) { val5 = data1[alu9+(-225)]; }
-        float val6 = 0.0f;
-        if ((alu12*alu13)) { val6 = data1[alu9+223]; }
-        float val7 = 0.0f;
-        if ((alu8*alu12*alu13)) { val7 = data1[alu9+671]; }
-        int alu14 = ((gidx1*1176)+(ridx0*49)+(ridx1*7)+ridx2);
-        float val8 = data2[alu14];
-        float val9 = data2[alu14+147];
-        float val10 = data2[alu14+294];
-        float val11 = data2[alu14+441];
-        float val12 = data2[alu14+588];
-        float val13 = data2[alu14+735];
-        float val14 = data2[alu14+882];
-        float val15 = data2[alu14+1029];
-        (acc0).x = ((val0*val8)+(acc0).x);
-        (acc1).x = ((val0*val9)+(acc1).x);
-        (acc2).x = ((val0*val10)+(acc2).x);
-        (acc3).x = ((val0*val11)+(acc3).x);
-        (acc4).x = ((val1*val8)+(acc4).x);
-        (acc5).x = ((val1*val9)+(acc5).x);
-        (acc6).x = ((val1*val10)+(acc6).x);
-        (acc7).x = ((val1*val11)+(acc7).x);
-        (acc8).x = ((val2*val8)+(acc8).x);
-        (acc9).x = ((val2*val9)+(acc9).x);
-        (acc10).x = ((val2*val10)+(acc10).x);
-        (acc11).x = ((val2*val11)+(acc11).x);
-        (acc12).x = ((val3*val8)+(acc12).x);
-        (acc13).x = ((val3*val9)+(acc13).x);
-        (acc14).x = ((val3*val10)+(acc14).x);
-        (acc15).x = ((val3*val11)+(acc15).x);
-        (acc16).x = ((val0*val12)+(acc16).x);
-        (acc17).x = ((val0*val13)+(acc17).x);
-        (acc18).x = ((val0*val14)+(acc18).x);
-        (acc19).x = ((val0*val15)+(acc19).x);
-        (acc20).x = ((val1*val12)+(acc20).x);
-        (acc21).x = ((val1*val13)+(acc21).x);
-        (acc22).x = ((val1*val14)+(acc22).x);
-        (acc23).x = ((val1*val15)+(acc23).x);
-        (acc24).x = ((val2*val12)+(acc24).x);
-        (acc25).x = ((val2*val13)+(acc25).x);
-        (acc26).x = ((val2*val14)+(acc26).x);
-        (acc27).x = ((val2*val15)+(acc27).x);
-        (acc28).x = ((val3*val12)+(acc28).x);
-        (acc29).x = ((val3*val13)+(acc29).x);
-        (acc30).x = ((val3*val14)+(acc30).x);
-        (acc31).x = ((val3*val15)+(acc31).x);
-        (acc0).y = ((val4*val8)+(acc0).y);
-        (acc1).y = ((val4*val9)+(acc1).y);
-        (acc2).y = ((val4*val10)+(acc2).y);
-        (acc3).y = ((val4*val11)+(acc3).y);
-        (acc4).y = ((val5*val8)+(acc4).y);
-        (acc5).y = ((val5*val9)+(acc5).y);
-        (acc6).y = ((val5*val10)+(acc6).y);
-        (acc7).y = ((val5*val11)+(acc7).y);
-        (acc8).y = ((val6*val8)+(acc8).y);
-        (acc9).y = ((val6*val9)+(acc9).y);
-        (acc10).y = ((val6*val10)+(acc10).y);
-        (acc11).y = ((val6*val11)+(acc11).y);
-        (acc12).y = ((val7*val8)+(acc12).y);
-        (acc13).y = ((val7*val9)+(acc13).y);
-        (acc14).y = ((val7*val10)+(acc14).y);
-        (acc15).y = ((val7*val11)+(acc15).y);
-        (acc16).y = ((val4*val12)+(acc16).y);
-        (acc17).y = ((val4*val13)+(acc17).y);
-        (acc18).y = ((val4*val14)+(acc18).y);
-        (acc19).y = ((val4*val15)+(acc19).y);
-        (acc20).y = ((val5*val12)+(acc20).y);
-        (acc21).y = ((val5*val13)+(acc21).y);
-        (acc22).y = ((val5*val14)+(acc22).y);
-        (acc23).y = ((val5*val15)+(acc23).y);
-        (acc24).y = ((val6*val12)+(acc24).y);
-        (acc25).y = ((val6*val13)+(acc25).y);
-        (acc26).y = ((val6*val14)+(acc26).y);
-        (acc27).y = ((val6*val15)+(acc27).y);
-        (acc28).y = ((val7*val12)+(acc28).y);
-        (acc29).y = ((val7*val13)+(acc29).y);
-        (acc30).y = ((val7*val14)+(acc30).y);
-        (acc31).y = ((val7*val15)+(acc31).y);
-      }
-    }
-  }
-  *((float2*)(data0+alu4)) = acc0;
-  *((float2*)(data0+alu4+12544)) = acc1;
-  *((float2*)(data0+alu4+25088)) = acc2;
-  *((float2*)(data0+alu4+37632)) = acc3;
-  *((float2*)(data0+alu4+112)) = acc4;
-  *((float2*)(data0+alu4+12656)) = acc5;
-  *((float2*)(data0+alu4+25200)) = acc6;
-  *((float2*)(data0+alu4+37744)) = acc7;
-  *((float2*)(data0+alu4+224)) = acc8;
-  *((float2*)(data0+alu4+12768)) = acc9;
-  *((float2*)(data0+alu4+25312)) = acc10;
-  *((float2*)(data0+alu4+37856)) = acc11;
-  *((float2*)(data0+alu4+336)) = acc12;
-  *((float2*)(data0+alu4+12880)) = acc13;
-  *((float2*)(data0+alu4+25424)) = acc14;
-  *((float2*)(data0+alu4+37968)) = acc15;
-  *((float2*)(data0+alu4+50176)) = acc16;
-  *((float2*)(data0+alu4+62720)) = acc17;
-  *((float2*)(data0+alu4+75264)) = acc18;
-  *((float2*)(data0+alu4+87808)) = acc19;
-  *((float2*)(data0+alu4+50288)) = acc20;
-  *((float2*)(data0+alu4+62832)) = acc21;
-  *((float2*)(data0+alu4+75376)) = acc22;
-  *((float2*)(data0+alu4+87920)) = acc23;
-  *((float2*)(data0+alu4+50400)) = acc24;
-  *((float2*)(data0+alu4+62944)) = acc25;
-  *((float2*)(data0+alu4+75488)) = acc26;
-  *((float2*)(data0+alu4+88032)) = acc27;
-  *((float2*)(data0+alu4+50512)) = acc28;
-  *((float2*)(data0+alu4+63056)) = acc29;
-  *((float2*)(data0+alu4+75600)) = acc30;
-  *((float2*)(data0+alu4+88144)) = acc31;
-}
-"""
-
-dev = "HIP"
-lib = Device[dev].compiler.compile(prefix+code)
-#lib = compile_hip(code)
-b0 = Buffer(dev, 1605632, dtypes.float)
-b1 = Buffer(dev, 301506, dtypes.float)
-b2 = Buffer(dev, 9408, dtypes.float)
-print(hex(b0._buf.value), hex(b0._buf.value+1605632*4))
-print(hex(b1._buf.value))
-print(hex(b2._buf.value))
-#prg = CompiledRunner("r_2_8_7_7_4_8_3_7_7_4_4_2_2", "", dev, [7, 1, 1], [8, 4, 1], precompiled=lib)
-prg = CompiledRunner("r_2_8_7_7_4_8_3_7_7_4_4_2_2", "", dev, [49, 8, 2], [8, 4, 1], precompiled=lib)
-print("compiled")
-prg([b0, b1, b2], {})
-print("ran")
-Device[dev].synchronize()
-print("sync")

test/external/external_test_am.py

@@ -2,7 +2,7 @@ import unittest
 from tinygrad.runtime.support.am.amdev import AMMemoryManager, AMPageTableEntry
 from tinygrad.runtime.support.am.ip import AM_GMC
 from tinygrad.runtime.support.hcq import MMIOInterface
-from tinygrad.runtime.support.memory import PageTableTraverseContext
+from tinygrad.runtime.support.memory import PageTableTraverseContext, AddrSpace
 from tinygrad.runtime.autogen.am import am
 from tinygrad.helpers import mv_address
 
@@ -70,7 +70,7 @@ class TestAMPageTable(unittest.TestCase):
 
     for va,sz in [(0x10000, 0x3000), (0x11000, 0x300000), (0x10000, 0x2000), (0x11000, 0x5000),
                   (0x2000000, 0x2000), (0x4000000, 0x4000000), (0x38000, 0x303000), (0x8000, 0x1000)]:
-      mm.map_range(vaddr=helper_va(va), size=sz, paddrs=[(va, sz)])
+      mm.map_range(vaddr=helper_va(va), size=sz, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       ctx = PageTableTraverseContext(self.d[0], mm.root_page_table, helper_va(va))
       results = list(ctx.next(sz))
@@ -102,8 +102,8 @@ class TestAMPageTable(unittest.TestCase):
     mm0 = self.d[0].mm
 
     for (va1,sz1),(va2,sz2) in [((0x10000, (0x1000)), (0x11000, (2 << 20)))]:
-      mm0.map_range(vaddr=helper_va(va1), size=sz1, paddrs=[(va1, sz1)])
-      mm0.map_range(vaddr=helper_va(va2), size=sz2, paddrs=[(va2, sz2)])
+      mm0.map_range(vaddr=helper_va(va1), size=sz1, paddrs=[(va1, sz1)], aspace=AddrSpace.PHYS)
+      mm0.map_range(vaddr=helper_va(va2), size=sz2, paddrs=[(va2, sz2)], aspace=AddrSpace.PHYS)
       mm0.unmap_range(helper_va(va2), sz2)
       mm0.unmap_range(helper_va(va1), sz1)
 
@@ -112,24 +112,24 @@ class TestAMPageTable(unittest.TestCase):
 
     for va,sz in [(0x10000, 0x3000), (0x1000000, 0x1000000), (0x12000, 0x4000)]:
       exteranl_va = helper_va(va)
-      mm0.map_range(vaddr=exteranl_va, size=sz, paddrs=[(va, sz)])
+      mm0.map_range(vaddr=exteranl_va, size=sz, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       with self.assertRaises(AssertionError):
-        mm0.map_range(vaddr=exteranl_va, size=0x1000, paddrs=[(va, sz)])
+        mm0.map_range(vaddr=exteranl_va, size=0x1000, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       with self.assertRaises(AssertionError):
-        mm0.map_range(vaddr=exteranl_va, size=0x100000, paddrs=[(va, sz)])
+        mm0.map_range(vaddr=exteranl_va, size=0x100000, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       with self.assertRaises(AssertionError):
-        mm0.map_range(vaddr=exteranl_va + 0x1000, size=0x1000, paddrs=[(va, sz)])
+        mm0.map_range(vaddr=exteranl_va + 0x1000, size=0x1000, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       with self.assertRaises(AssertionError):
-        mm0.map_range(vaddr=exteranl_va + 0x2000, size=0x100000, paddrs=[(va, sz)])
+        mm0.map_range(vaddr=exteranl_va + 0x2000, size=0x100000, paddrs=[(va, sz)], aspace=AddrSpace.PHYS)
 
       mm0.unmap_range(vaddr=exteranl_va, size=sz)
 
       # Finally can map and check paddrs
-      mm0.map_range(vaddr=exteranl_va + 0x2000, size=0x100000, paddrs=[(0xdead0000, 0x1000), (0xdead1000, 0xff000)])
+      mm0.map_range(vaddr=exteranl_va + 0x2000, size=0x100000, paddrs=[(0xdead0000, 0x1000), (0xdead1000, 0xff000)], aspace=AddrSpace.PHYS)
 
       ctx = PageTableTraverseContext(self.d[0], mm0.root_page_table, exteranl_va + 0x2000)
       for tup in ctx.next(0x100000):
@@ -147,13 +147,13 @@ class TestAMPageTable(unittest.TestCase):
     with self.assertRaises(AssertionError):
       mm0.unmap_range(helper_va(0x10000), 0x3000)
 
-    mm0.map_range(helper_va(0x10000), 0x3000, paddrs=[(0x10000, 0x3000)])
+    mm0.map_range(helper_va(0x10000), 0x3000, paddrs=[(0x10000, 0x3000)], aspace=AddrSpace.PHYS)
     mm0.unmap_range(helper_va(0x10000), 0x3000)
 
     with self.assertRaises(AssertionError):
       mm0.unmap_range(helper_va(0x10000), 0x3000)
 
-    mm0.map_range(helper_va(0x10000), 0x3000, paddrs=[(0x10000, 0x3000)])
+    mm0.map_range(helper_va(0x10000), 0x3000, paddrs=[(0x10000, 0x3000)], aspace=AddrSpace.PHYS)
     mm0.unmap_range(helper_va(0x10000), 0x3000)
 
     with self.assertRaises(AssertionError):
@@ -164,16 +164,16 @@ class TestAMPageTable(unittest.TestCase):
 
     # offset from start
     for off in [0, 0x3000, 0x10000]:
-      mm0.map_range(helper_va(0x1000000) + off, (2 << 20)  - off, paddrs=[(0x10000, 0x1000)] * (512 - off // 0x1000))
+      mm0.map_range(helper_va(0x1000000) + off, (2 << 20)  - off, paddrs=[(0x10000, 0x1000)] * (512 - off // 0x1000), aspace=AddrSpace.PHYS)
       mm0.unmap_range(helper_va(0x1000000) + off, (2 << 20) - off)
-      mm0.map_range(helper_va(0x1000000), 2 << 20, paddrs=[(0x10000, 2 << 20)])
+      mm0.map_range(helper_va(0x1000000), 2 << 20, paddrs=[(0x10000, 2 << 20)], aspace=AddrSpace.PHYS)
       mm0.unmap_range(helper_va(0x1000000), 2 << 20)
 
     # offset from end
     for off in [0x1000, 0x20000]:
-      mm0.map_range(helper_va(0x1000000), (2 << 20) - off, paddrs=[(0x10000, 0x1000)] * (512 - off // 0x1000))
+      mm0.map_range(helper_va(0x1000000), (2 << 20) - off, paddrs=[(0x10000, 0x1000)] * (512 - off // 0x1000), aspace=AddrSpace.PHYS)
       mm0.unmap_range(helper_va(0x1000000), (2 << 20) - off)
-      mm0.map_range(helper_va(0x1000000), 2 << 20, paddrs=[(0x10000, 2 << 20)])
+      mm0.map_range(helper_va(0x1000000), 2 << 20, paddrs=[(0x10000, 2 << 20)], aspace=AddrSpace.PHYS)
       mm0.unmap_range(helper_va(0x1000000), 2 << 20)
 
   def test_frag_size(self):

test/external/external_test_nv.py

@@ -1,60 +0,0 @@
-import unittest, struct, array, ctypes
-from tinygrad import Device, dtypes, Tensor
-from tinygrad.helpers import to_mv
-from tinygrad.runtime.ops_nv import NVDevice, HWQueue
-from tinygrad.codegen.opt.search import Opt, OptOps
-from tinygrad.engine.realize import get_runner, CompiledRunner, get_program
-from test.external.fuzz_linearizer import get_fuzz_rawbufs
-
-from tinygrad.codegen.opt.kernel import Kernel
-from tinygrad.uop.ops import LazyOp, Ops, ReduceOps, BufferOps, MemBuffer
-from tinygrad.shape.shapetracker import ShapeTracker
-from tinygrad.shape.view import View
-
-@unittest.skipUnless(Device.DEFAULT == "NV", "NV specific tests/fixes")
-class TestNV(unittest.TestCase):
-  @classmethod
-  def setUpClass(self):
-    TestNV.d0: NVDevice = Device["NV"]
-    TestNV.a = Tensor([0.,1.], device="NV").realize()
-    TestNV.b = self.a + 1
-    si = self.b.schedule()[-1]
-    TestNV.d0_runner = get_runner(TestNV.d0.device, si.ast)
-    TestNV.b.uop.buffer.allocate()
-    TestNV.addr = struct.pack("QQ", TestNV.b.uop.buffer._buf.va_addr, TestNV.a.uop.buffer._buf.va_addr)
-
-  def test_error_on_huge_dims(self):
-    ast = LazyOp(op=BufferOps.STORE, src=(LazyOp(op=ReduceOps.SUM, src=(LazyOp(op=Ops.CAST, src=(LazyOp(op=Ops.MUL, src=(LazyOp(op=BufferOps.LOAD, src=(), arg=MemBuffer(idx=1, dtype=dtypes.half, st=ShapeTracker(views=(View(shape=(1, 1, 1024, 683), strides=(0, 0, 0, 1), offset=0, mask=None, contiguous=False),)))), LazyOp(op=BufferOps.LOAD, src=(), arg=MemBuffer(idx=2, dtype=dtypes.half, st=ShapeTracker(views=(View(shape=(1, 1, 1024, 683), strides=(0, 0, 683, 1), offset=0, mask=None, contiguous=True),))))), arg=None),), arg=dtypes.float),), arg=(3,)),), arg=MemBuffer(idx=0, dtype=dtypes.float, st=ShapeTracker(views=(View(shape=(1, 1, 1024, 1), strides=(0, 0, 1, 0), offset=0, mask=None, contiguous=True),)))) # noqa: E501
-    opts = [Opt(op=OptOps.GROUP, axis=0, arg=0), Opt(op=OptOps.PADTO, axis=1, arg=32), Opt(op=OptOps.UNROLL, axis=0, arg=4), Opt(op=OptOps.LOCAL, axis=0, arg=2), Opt(op=OptOps.LOCAL, axis=0, arg=2)] # noqa: E501
-    with self.assertRaises(RuntimeError) as cm:
-      lin = Kernel(ast)
-      lin.apply_opts(opts)
-      rawbufs = get_fuzz_rawbufs(lin)
-      prg = CompiledRunner(get_program(lin.get_optimized_ast(), lin.opts))
-      prg(rawbufs, {}, wait=True)
-    self.assertEqual(str(cm.exception), "This is a runtime error message")
-
-  def test_buf4_usage(self):
-    TestNV.along = Tensor([105615], device="NV").realize()
-    ast = LazyOp(op=BufferOps.STORE, src=(LazyOp(op=Ops.SIN, src=(LazyOp(op=Ops.CAST, src=(LazyOp(op=BufferOps.LOAD, src=(), arg=MemBuffer(idx=1, dtype=dtypes.ulong, st=ShapeTracker(views=(View(shape=(3,), strides=(1,), offset=0, mask=None, contiguous=True),)))),), arg=dtypes.float),), arg=None),), arg=MemBuffer(idx=0, dtype=dtypes.float, st=ShapeTracker(views=(View(shape=(3,), strides=(1,), offset=0, mask=None, contiguous=True),)))) # noqa: E501
-    temp_runner = get_runner(TestNV.d0.device, (ast,))
-    temp_runner([TestNV.b.uop.buffer, TestNV.along.uop.buffer], var_vals={})
-    val = TestNV.b.uop.buffer.as_buffer().cast("f")[0]
-    assert abs(val - 0.80647) < 0.001, f"got val {val}"
-
-  def test_kernargs_no_oob_access(self):
-    kernargs_start = TestNV.d0._gpu_alloc((2 << 20), map_to_cpu=True).va_addr
-    kernargs = kernargs_start + ((2 << 20) - TestNV.d0_runner._prg.kernargs_alloc_size)
-    to_mv(kernargs, 0x160).cast('I')[:] = array.array('I', TestNV.d0_runner._prg.constbuffer_0)
-    ctypes.memmove(kernargs + TestNV.d0_runner._prg.kernargs_offset, TestNV.addr, len(TestNV.addr))
-
-    q = HWQueue()
-    q.exec(TestNV.d0_runner._prg, kernargs, TestNV.d0_runner.global_size, TestNV.d0_runner.local_size)
-    q.signal(TestNV.d0.timeline_signal, TestNV.d0.timeline_value).submit(TestNV.d0)
-    TestNV.d0._wait_signal(TestNV.d0.timeline_signal, TestNV.d0.timeline_value)
-    TestNV.d0.timeline_value += 1
-    val = TestNV.b.uop.buffer.as_buffer().cast("f")[0]
-    assert val == 1.0, f"got val {val}"
-
-if __name__ == "__main__":
-  unittest.main()

test/external/fuzz_shapetracker.py

@@ -1,61 +0,0 @@
-import random
-from tinygrad.helpers import DEBUG, getenv
-from test.unit.test_shapetracker import CheckingShapeTracker
-
-def do_permute(st):
-  perm = list(range(0, len(st.shape)))
-  random.shuffle(perm)
-  perm = tuple(perm)
-  if DEBUG >= 1: print("st.permute(", perm, ")")
-  st.permute(perm)
-
-def do_pad(st):
-  c = random.randint(0, len(st.shape)-1)
-  pad = tuple((random.randint(0,2), random.randint(0,2)) if i==c else (0,0) for i in range(len(st.shape)))
-  if DEBUG >= 1: print("st.pad(", pad, ")")
-  st.pad(pad)
-
-def do_reshape_split_one(st):
-  c = random.randint(0, len(st.shape)-1)
-  poss = [n for n in [1,2,3,4,5] if st.shape[c]%n == 0]
-  spl = random.choice(poss)
-  shp = st.shape[0:c] + (st.shape[c]//spl, spl) + st.shape[c+1:]
-  if DEBUG >= 1: print("st.reshape(", shp, ")")
-  st.reshape(shp)
-
-def do_reshape_combine_two(st):
-  if len(st.shape) < 2: return
-  c = random.randint(0, len(st.shape)-2)
-  shp = st.shape[:c] + (st.shape[c] * st.shape[c+1], ) + st.shape[c+2:]
-  if DEBUG >= 1: print("st.reshape(", shp, ")")
-  st.reshape(shp)
-
-def do_shrink(st):
-  c = random.randint(0, len(st.shape)-1)
-  while 1:
-    shrink = tuple((random.randint(0,s), random.randint(0,s)) if i == c else (0,s) for i,s in enumerate(st.shape))
-    if all(x<y for (x,y) in shrink): break
-  if DEBUG >= 1: print("st.shrink(", shrink, ")")
-  st.shrink(shrink)
-
-def do_flip(st):
-  flip = tuple(random.random() < 0.5 for _ in st.shape)
-  if DEBUG >= 1: print("st.flip(", flip, ")")
-  st.flip(flip)
-
-def do_expand(st):
-  c = [i for i,s in enumerate(st.shape) if s==1]
-  if len(c) == 0: return
-  c = random.choice(c)
-  expand = tuple(random.choice([2,3,4]) if i==c else s for i,s in enumerate(st.shape))
-  if DEBUG >= 1: print("st.expand(", expand, ")")
-  st.expand(expand)
-
-shapetracker_ops = [do_permute, do_pad, do_shrink, do_reshape_split_one, do_reshape_combine_two, do_flip, do_expand]
-
-if __name__ == "__main__":
-  random.seed(42)
-  for _ in range(getenv("CNT", 200)):
-    st = CheckingShapeTracker((random.randint(2, 10), random.randint(2, 10), random.randint(2, 10)))
-    for i in range(8): random.choice(shapetracker_ops)(st)
-    st.assert_same()

test/external/fuzz_shapetracker_math.py

@@ -1,34 +0,0 @@
-import random
-from tinygrad.helpers import getenv, DEBUG, colored, trange
-from tinygrad.shape.shapetracker import ShapeTracker
-from test.external.fuzz_shapetracker import shapetracker_ops
-from test.unit.test_shapetracker_math import st_equal, MultiShapeTracker
-
-def fuzz_plus() -> tuple[ShapeTracker, ShapeTracker]:
-  m = MultiShapeTracker([ShapeTracker.from_shape((random.randint(1, 10), random.randint(1, 10), random.randint(1, 10)))])
-  for _ in range(4): random.choice(shapetracker_ops)(m)
-  backup = m.sts[0]
-  m.sts.append(ShapeTracker.from_shape(m.sts[0].shape))
-  for _ in range(4): random.choice(shapetracker_ops)(m)
-  st_sum = backup + m.sts[1]
-  return m.sts[0], st_sum
-
-if __name__ == "__main__":
-  if seed:=getenv("SEED"): random.seed(seed)
-  total = getenv("CNT", 1000)
-  for fuzz in [globals()[f'fuzz_{x}'] for x in getenv("FUZZ", "plus").split(",")]:
-    same_but_neq = 0
-    for _ in trange(total, desc=f"{fuzz}"):
-      st1, st2 = fuzz()
-      eq = st_equal(st1, st2)
-      if getenv("CHECK_NEQ") and eq and st1.simplify() != st2.simplify():
-        print(colored("same but unequal", "yellow"))
-        print(st1.simplify())
-        print(st2.simplify())
-        same_but_neq += 1
-      if DEBUG >= 1:
-        print(f"EXP: {st1}")
-        print(f"GOT: {st2}")
-        print(colored("****", "green" if eq else "red"))
-      if not eq: exit(0)
-    if getenv("CHECK_NEQ"): print(f"same but unequal {same_but_neq}/{total} = {(same_but_neq/total)*100:.2f}%")

test/helpers.py

@@ -2,18 +2,27 @@ import os, time, struct, functools, unittest
 from typing import Any, Callable
 import numpy as np
 from tinygrad import Tensor, dtypes, Device
-from tinygrad.uop.ops import UOp, Ops
+from tinygrad.uop.ops import UOp, Ops, KernelInfo
 from tinygrad.tensor import _to_np_dtype
-from tinygrad.engine.realize import Runner
+from tinygrad.engine.realize import Runner, get_program
 from tinygrad.dtype import DType
 from tinygrad.nn.state import get_parameters
 from tinygrad.helpers import T, CI
-from tinygrad.codegen import full_rewrite
+from tinygrad.renderer import Renderer
+from tinygrad.codegen import full_rewrite_to_sink, line_rewrite, pm_linearize_cleanups
+from tinygrad.codegen.late.linearizer import linearize
 
 # decorator to skip slow tests by default, run with RUN_SLOW=1 to include them
 slow = unittest.skipUnless(os.getenv("RUN_SLOW"), "slow test, set RUN_SLOW=1 to run")
 from tinygrad.runtime.ops_python import PythonProgram, PythonRenderer, PythonCompiler
 
+def get_uops(sink:UOp, ren:Renderer|None=None) -> list[UOp]:
+  """Extract linearized UOps from a sink. Test helper that only does linearization (no render)."""
+  if ren is None: ren = Renderer()
+  if sink.arg is None: sink = sink.replace(arg=KernelInfo())
+  full_sink = full_rewrite_to_sink(sink, ren, optimize=sink.tag is None)
+  return line_rewrite(linearize(full_sink), pm_linearize_cleanups)
+
 def derandomize_model(model):
   for p in get_parameters(model):
     p.replace(Tensor.empty(p.shape, device=p.device, dtype=p.dtype))
@@ -51,13 +60,12 @@ def eval_uop(uop:UOp, inputs:list[tuple[DType, list[Any]]]|None=None):
   bufs = []
   for buf_dt, data in inputs or []:
     bufs.append(buf:=allocator.alloc(len(data) * buf_dt.itemsize))
-    allocator._copyin(buf, memoryview(struct.pack(str(len(data)) + buf_dt.fmt, *data)))
+    allocator._copyin(buf, memoryview(struct.pack(str(len(data)) + (buf_dt.fmt or ""), *data)))
   g = UOp(Ops.DEFINE_GLOBAL, uop.dtype.ptr(), arg=0, src=())
-  opts = PythonRenderer()
-  lst = full_rewrite(UOp.store(g.index(UOp.const(dtypes.int, 0)), uop).sink(), opts)
-  prog = PythonProgram("run", PythonCompiler().compile(opts.render(lst)))
+  prg = get_program(UOp.store(g.index(UOp.const(dtypes.int, 0)), uop).sink(), PythonRenderer())
+  prog = PythonProgram("run", PythonCompiler().compile(prg.src))
   prog(out_buf:=allocator.alloc(uop.dtype.itemsize), *bufs)
-  return out_buf.cast(uop.dtype.fmt).tolist()[0]
+  return out_buf.cast(uop.dtype.fmt or "").tolist()[0]
 
 def not_support_multi_device():
   # CL and CUDA don't support multi device if in CI

test/mockgpu/amd/amddriver.py

@@ -52,6 +52,7 @@ class AMDDriver(VirtDriver):
     self.doorbells = {}
     self.next_doorbell = collections.defaultdict(int)
     self.mmu_event_ids = []
+    self._executing = False  # re-entrancy guard for _emulate_execute
 
     for i in range(gpus): self._prepare_gpu(i+1)
 
@@ -125,6 +126,9 @@ class AMDDriver(VirtDriver):
       if struct.gpu_id not in self.gpus: return -1
       struct.handle = self._alloc_handle()
       self.object_by_handle[struct.handle] = copy.deepcopy(struct) # save memory struct to know what mem it is
+      # Track signal memory (uncached + coherent) - progress queues when written to
+      if struct.flags & kfd.KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED:
+        self.track_address(struct.va_addr, struct.va_addr + struct.size, lambda mv,off: None, lambda mv, off: self._emulate_execute())
     elif nr == kfd_ioctls.AMDKFD_IOC_FREE_MEMORY_OF_GPU:
       self.object_by_handle.pop(struct.handle)
     elif nr == kfd_ioctls.AMDKFD_IOC_MAP_MEMORY_TO_GPU:
@@ -173,9 +177,14 @@ class AMDDriver(VirtDriver):
     return 0
 
   def _emulate_execute(self):
-    any_progress = True
-    while any_progress:
-      any_progress = False
-      for gpu in self.gpus.values():
-        for q in gpu.queues:
-          if q.executing: any_progress |= q.execute() > 0
+    if self._executing: return  # prevent re-entrancy
+    self._executing = True
+    try:
+      any_progress = True
+      while any_progress:
+        any_progress = False
+        for gpu in self.gpus.values():
+          for q in gpu.queues:
+            if q.executing: any_progress |= q.execute() > 0
+    finally:
+      self._executing = False

test/mockgpu/amd/amdgpu.py

@@ -7,6 +7,7 @@ import tinygrad.runtime.autogen.amd_gpu as amd_gpu, tinygrad.runtime.autogen.am.
 SDMA_MAX_COPY_SIZE = 0x400000
 
 regCOMPUTE_PGM_LO = 0x1bac + amd_gpu.GC_BASE__INST0_SEG0
+regCOMPUTE_PGM_RSRC2 = 0x1bb3 + amd_gpu.GC_BASE__INST0_SEG0
 regCOMPUTE_USER_DATA_0 = 0x1be0 + amd_gpu.GC_BASE__INST0_SEG0
 regCOMPUTE_NUM_THREAD_X = 0x1ba7 + amd_gpu.GC_BASE__INST0_SEG0
 regGRBM_GFX_INDEX = 0x2200 + amd_gpu.GC_BASE__INST0_SEG1
@@ -179,12 +180,16 @@ class PM4Executor(AMDQueue):
     prg_addr = (self.gpu.regs[regCOMPUTE_PGM_LO] + (self.gpu.regs[regCOMPUTE_PGM_LO + 1] << 32)) << 8
     args_addr = self.gpu.regs[regCOMPUTE_USER_DATA_0] + (self.gpu.regs[regCOMPUTE_USER_DATA_0 + 1] << 32)
     lc = [self.gpu.regs[i] for i in range(regCOMPUTE_NUM_THREAD_X, regCOMPUTE_NUM_THREAD_X+3)]
+    rsrc2 = self.gpu.regs[regCOMPUTE_PGM_RSRC2]
 
     prg_sz = 0
     for st,sz in self.gpu.mapped_ranges:
       if st <= prg_addr < st+sz: prg_sz = sz - (prg_addr - st)
 
     assert prg_sz > 0, "Invalid prg ptr (not found in mapped ranges)"
+    # Pass valid memory ranges and rsrc2 to Python emulator for bounds checking and SGPR layout
+    if hasattr(remu, 'valid_mem_ranges'): remu.valid_mem_ranges = self.gpu.mapped_ranges
+    if hasattr(remu, 'rsrc2'): remu.rsrc2 = rsrc2
     err = remu.run_asm(prg_addr, prg_sz, *gl, *lc, args_addr)
     if err != 0: raise RuntimeError("remu does not support the new instruction introduced in this kernel")
 

test/mockgpu/helpers.py

@@ -1,4 +1,5 @@
 import ctypes, ctypes.util
+from tinygrad.helpers import getenv
 
 def _try_dlopen_gpuocelot():
   GPUOCELOT_PATHS = [ctypes.util.find_library("gpuocelot")] if ctypes.util.find_library("gpuocelot") is not None else []
@@ -14,7 +15,21 @@ def _try_dlopen_gpuocelot():
   print("Could not find libgpuocelot.so")
   return None
 
+class PythonRemu:
+  """Python RDNA3 emulator wrapper that matches the libremu.so interface."""
+  valid_mem_ranges: set[tuple[int, int]] = set()
+  rsrc2: int = 0x19c  # Default: USER_SGPR_COUNT=14, enable X and Y workgroup IDs
+
+  def run_asm(self, lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int) -> int:
+    from extra.assembly.amd.emu import run_asm, set_valid_mem_ranges
+    # Pad ranges to handle GPU loads that may read past small buffers (e.g. s_load_b128 on 12-byte buffer)
+    set_valid_mem_ranges({(start, size + 4096) for start, size in self.valid_mem_ranges})
+    return run_asm(lib, lib_sz, gx, gy, gz, lx, ly, lz, args_ptr, self.rsrc2)
+
 def _try_dlopen_remu():
+  # Use Python emulator only if PYTHON_REMU=1
+  if getenv("PYTHON_REMU"):
+    return PythonRemu()
   REMU_PATHS = ["extra/remu/target/release/libremu.so", "libremu.so", "/usr/local/lib/libremu.so",
                "extra/remu/target/release/libremu.dylib", "libremu.dylib", "/usr/local/lib/libremu.dylib", "/opt/homebrew/lib/libremu.dylib"]
   for path in REMU_PATHS:

test/mockgpu/nv/nvdriver.py

@@ -15,7 +15,7 @@ libc.munmap.restype = ctypes.c_int
 NVSubDevice = collections.namedtuple('NVSubDevice', ['device'])
 NVUserMode = collections.namedtuple('NVUserMode', ['subdevice'])
 NVVASpace = collections.namedtuple('NVVASpace', ['device'])
-NVAllocation = collections.namedtuple('NVAllocation', ['device', 'size'])
+NVAllocation = collections.namedtuple('NVAllocation', ['device', 'size', 'is_signal'])
 NVChannelGroup = collections.namedtuple('NVChannelGroup', ['device'])
 NVContextShare = collections.namedtuple('NVContextShare', ['channel_group'])
 NVGPFIFO = collections.namedtuple('NVGPFIFO', ['device', 'token'])
@@ -41,12 +41,14 @@ class NVDevFileDesc(VirtFileDesc):
     super().__init__(fd)
     self.driver, self.gpu = driver, gpu
     self._mapping_userland = False
+    self._mapping_signal = False
 
   def ioctl(self, fd, request, argp): return self.driver.dev_ioctl(self.gpu, request, argp)
   def mmap(self, start, sz, prot, flags, fd, offset):
     start = libc.mmap(start, sz, prot, flags|mmap.MAP_ANONYMOUS, -1, 0)
-    if self._mapping_userland:
+    if self._mapping_userland or self._mapping_signal:
       self.driver.track_address(start, start+sz, lambda mv,off: None, lambda mv, off: self.driver._gpu_mmio_write(mv, off, self.gpu))
+      self._mapping_signal = False
     return start
 
 class NVDriver(VirtDriver):
@@ -65,6 +67,7 @@ class NVDriver(VirtDriver):
     self.object_by_handle = {}
     self.opened_fds = {}
     self.next_doorbell = collections.defaultdict(int)
+    self._executing = False  # re-entrancy guard for _gpu_mmio_write
 
     for i in range(gpus): self._prepare_gpu(i)
 
@@ -115,7 +118,8 @@ class NVDriver(VirtDriver):
       assert struct.hObjectParent in self.object_by_handle and isinstance(self.object_by_handle[struct.hObjectParent], NVGPU)
       params = nv_gpu.NV_MEMORY_ALLOCATION_PARAMS.from_address(params_ptr)
       struct.hObjectNew = self._alloc_handle()
-      self.object_by_handle[struct.hObjectNew] = NVAllocation(self.object_by_handle[struct.hObjectParent], params.size)
+      is_signal = struct.hClass == nv_gpu.NV1_MEMORY_SYSTEM  # signal memory uses NV1_MEMORY_SYSTEM (uncached)
+      self.object_by_handle[struct.hObjectNew] = NVAllocation(self.object_by_handle[struct.hObjectParent], params.size, is_signal)
     elif struct.hClass == nv_gpu.KEPLER_CHANNEL_GROUP_A:
       assert struct.hObjectParent in self.object_by_handle and isinstance(self.object_by_handle[struct.hObjectParent], NVGPU)
       struct.hObjectNew = self._alloc_handle()
@@ -206,7 +210,6 @@ class NVDriver(VirtDriver):
   def ctl_ioctl(self, req, argp):
     nr = req & 0xff
     if nr == nv_gpu.NV_ESC_RM_ALLOC: return self.rm_alloc(argp)
-    elif nr == nv_gpu.NV_ESC_RM_ALLOC_MEMORY: pass
     elif nr == nv_gpu.NV_ESC_RM_CONTROL: return self.rm_control(argp)
     elif nr == nv_gpu.NV_ESC_RM_MAP_MEMORY:
       st:Any = nv_gpu.nv_ioctl_nvos33_parameters_with_fd.from_address(argp)
@@ -215,6 +218,10 @@ class NVDriver(VirtDriver):
         file = self.opened_fds[st.fd]
         assert isinstance(file, NVDevFileDesc)
         file._mapping_userland = True
+      elif isinstance(obj, NVAllocation) and obj.is_signal:
+        file = self.opened_fds[st.fd]
+        assert isinstance(file, NVDevFileDesc)
+        file._mapping_signal = True
     elif nr == nv_gpu.NV_ESC_RM_FREE:
       st = nv_gpu.NVOS00_PARAMETERS.from_address(argp)
       self.object_by_handle.pop(st.hObjectOld)
@@ -256,12 +263,26 @@ class NVDriver(VirtDriver):
     else: raise RuntimeError(f"Unknown {nr} to nvidia-uvm")
     return 0
 
-  def dev_ioctl(self, dev, req, argp): return 0
+  def dev_ioctl(self, dev, req, argp):
+    nr = req & 0xff
+    # Handle NV_ESC_RM_ALLOC_MEMORY for host/signal memory
+    if nr == nv_gpu.NV_ESC_RM_ALLOC_MEMORY:
+      st:Any = nv_gpu.nv_ioctl_nvos02_parameters_with_fd.from_address(argp)
+      # Track host memory (signal memory) - progress queues when written to
+      if st.params.hClass == nv_gpu.NV01_MEMORY_SYSTEM_OS_DESCRIPTOR:
+        self.track_address(st.params.pMemory, st.params.pMemory + st.params.limit + 1,
+                           lambda mv,off: None, lambda mv, off: self._gpu_mmio_write(mv, off, None))
+    return 0
   def _gpu_mmio_write(self, mv, off, gpu):
-    any_progress = True
-    while any_progress:
-      any_progress = False
-      for gpu in self.gpus.values():
-        for q in gpu.queues:
-          if q.ctrl.GPGet != q.ctrl.GPPut:
-            any_progress |= q.execute()
+    if self._executing: return  # prevent re-entrancy
+    self._executing = True
+    try:
+      any_progress = True
+      while any_progress:
+        any_progress = False
+        for gpu in self.gpus.values():
+          for q in gpu.queues:
+            if q.ctrl.GPGet != q.ctrl.GPPut:
+              any_progress |= q.execute()
+    finally:
+      self._executing = False

test/models/test_bert.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+import os
+os.environ['USE_TF'] = '0'  # prevent transformers from importing tensorflow
 import unittest
 from tinygrad import Tensor
 import numpy as np

test/models/test_efficientnet.py

@@ -68,7 +68,7 @@ class TestEfficientNet(unittest.TestCase):
     self.assertEqual(_LABELS[labels[0]], "sports car, sport car")
 
   def test_chicken_car(self):
-    labels = _infer(self.model, np.concat([chicken_img, car_img], axis=0))
+    labels = _infer(self.model, np.concatenate([chicken_img, car_img], axis=0))
     self.assertEqual(_LABELS[labels[0]], "hen")
     self.assertEqual(_LABELS[labels[1]], "sports car, sport car")
 

test/test_custom_kernel.py

@@ -193,6 +193,16 @@ class TestCustomKernel(unittest.TestCase):
     err = (tst - (a@b)).square().max()
     self.assertLess(err.item(), 1e-6)
 
+  def test_gemm_multi(self):
+    devs = ("CPU:0", "CPU:1")
+    N = 16
+    a = Tensor.randn(N, N).shard_(devs, axis=0)
+    b = Tensor.randn(N, N).to(devs)
+    c = Tensor(Tensor.empty(N//2, N, device=devs).uop.multi(0), device=devs)
+    tst = Tensor.custom_kernel(c, a, b, fxn=custom_gemm)[0]
+    err = (tst - (a@b)).square().max()
+    self.assertLess(err.item(), 1e-6)
+
   def test_gemm_backward_custom(self): self.test_gemm_backward(True)
   # NOTE: grad_fxn doesn't work with pyrender
   def test_gemm_backward(self, custom_backward_gemm=False):

test/test_dtype_alu.py

@@ -9,7 +9,7 @@ from tinygrad.renderer.ptx import PTXRenderer
 from tinygrad.renderer.nir import NIRRenderer
 import numpy as np
 import pytest
-from hypothesis import assume, given, strategies as strat, settings, HealthCheck
+from hypothesis import assume, given, strategies as strat, settings
 
 pytestmark = pytest.mark.filterwarnings("ignore")
 
@@ -206,29 +206,23 @@ class TestDTypeALU(unittest.TestCase):
   @given(ht.int32, strat.sampled_from(dtypes_float+dtypes_int+dtypes_bool))
   def test_int32_cast(self, a, dtype): universal_test_cast(a, dtypes.int32, dtype)
 
-  @settings(suppress_health_check=[HealthCheck.filter_too_much])
-  @given(strat.data(), strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
+  @given(strat.floats(width=32, min_value=1.0, max_value=254.0, allow_subnormal=False),
+         strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
   def test_float_cast_to_unsigned(self, a, float_dtype, unsigned_dtype):
     if not is_dtype_supported(float_dtype): float_dtype = dtypes.float32
-    float_strat = {dtypes.float16: ht.float16, dtypes.float32: ht.float32, dtypes.float64: ht.float64}[float_dtype]
-    float_strat = float_strat.filter(lambda x: 0 < x < dtypes.max(unsigned_dtype))
-    universal_test_cast(a.draw(float_strat), float_dtype, unsigned_dtype)
+    universal_test_cast(a, float_dtype, unsigned_dtype)
 
-  @settings(suppress_health_check=[HealthCheck.filter_too_much])
-  @given(strat.data(), strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
+  @given(strat.floats(width=32, min_value=256.0, max_value=65000.0, allow_subnormal=False),
+         strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
   def test_float_cast_to_unsigned_overflow(self, a, float_dtype, unsigned_dtype):
     if not is_dtype_supported(float_dtype): float_dtype = dtypes.float32
-    float_strat = {dtypes.float16: ht.float16, dtypes.float32: ht.float32, dtypes.float64: ht.float64}[float_dtype]
-    overflow_strat = float_strat.filter(lambda x: x > dtypes.max(unsigned_dtype) and x <= dtypes.max(dtypes.int32))
-    universal_test_cast(a.draw(overflow_strat), float_dtype, unsigned_dtype)
+    universal_test_cast(a, float_dtype, unsigned_dtype)
 
-  @settings(suppress_health_check=[HealthCheck.filter_too_much])
-  @given(strat.data(), strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
+  @given(strat.floats(width=32, min_value=-65000.0, max_value=-1.0, allow_subnormal=False),
+         strat.sampled_from(dtypes_float), strat.sampled_from((dtypes.uint8, dtypes.uint16)))
   def test_float_cast_to_unsigned_underflow(self, a, float_dtype, unsigned_dtype):
     if not is_dtype_supported(float_dtype): float_dtype = dtypes.float32
-    float_strat = {dtypes.float16: ht.float16, dtypes.float32: ht.float32, dtypes.float64: ht.float64}[float_dtype]
-    underflow_strat = float_strat.filter(lambda x: x < 0 and x >= dtypes.min(dtypes.int32))
-    universal_test_cast(a.draw(underflow_strat), float_dtype, unsigned_dtype)
+    universal_test_cast(a, float_dtype, unsigned_dtype)
 
   @unittest.expectedFailure
   def test_unsafe_cast_float_to_int_failure(self):

test/test_multitensor.py

@@ -57,12 +57,18 @@ class TestMultiTensor(unittest.TestCase):
       assert lb.shape == (128,)
     (X + X).realize()
 
+  @unittest.expectedFailure # TODO: fix
   def test_shard_empty(self):
     GlobalCounters.reset()
     X = Tensor.empty(256).shard(devices_2, 0).realize()
     assert GlobalCounters.kernel_count == 0
     (X + X).realize()
 
+  def test_arange_shrink(self):
+    x = Tensor.arange(4)
+    self.assertEqual(x.shard(devices_2, 0).realize().shrink(((2, 4),)).tolist(), [2, 3])
+    self.assertEqual(x.shard(devices_2, 0).realize().shrink(((0, 2),)).tolist(), [0, 1])
+
   def test_shard_like(self):
     X = Tensor.ones(256).shard(devices_2, 0)
     Y = Tensor.zeros(256).shard_like(X)

test/test_renderer_failures.py

@@ -1,30 +1,26 @@
 import unittest
 import numpy as np
+from dataclasses import replace
 from tinygrad.device import Buffer, Device, is_dtype_supported
 from tinygrad.dtype import dtypes, ConstType
-from tinygrad.engine.realize import CompiledRunner
-from tinygrad.helpers import dedup, flatten, prod
+from tinygrad.engine.realize import CompiledRunner, get_program
+from tinygrad.helpers import prod
 from tinygrad.renderer.cstyle import CStyleLanguage
 from tinygrad.renderer.ptx import PTXRenderer
 from tinygrad.renderer.wgsl import WGSLRenderer
 from tinygrad.runtime.ops_python import PythonRenderer
 from tinygrad.uop.ops import UOp, Ops, python_alu
-from tinygrad.renderer import ProgramSpec
 from tinygrad.tensor import Tensor, _to_np_dtype
-from tinygrad.codegen import full_rewrite
 
-def _test_uop_result(inputs:list[Tensor], stores:list[UOp], local_size=None):
+def _test_uop_result(inputs:list[Tensor], prg, local_size=None):
   for x in inputs: x.realize()
-  # NOTE: we only toposort the stores
-  uops: list[UOp] = []
-  def _recursive_add(uop:UOp) -> list[UOp]: return flatten([_recursive_add(x) for x in uop.src])+[uop]
-  uops = dedup(flatten(_recursive_add(st) for st in stores))
+  uops = prg.uops
   outbufs = [Buffer(Device.DEFAULT, sz:=(1 if local_size is None else prod(local_size)), (dtype:=u.src[1].dtype), \
       initial_value=np.zeros(sz, dtype=_to_np_dtype(dtype)).data) for u in uops if u.op is Ops.STORE]
   inbufs = [x.uop.base.buffer for x in inputs]
-  src = Device[Device.DEFAULT].renderer.render(uops)
-  ei = CompiledRunner(ProgramSpec(uops[-1].arg.name if uops[-1].arg is not None else "test",
-                                  src, Device.DEFAULT, uops[-1], uops=uops, local_size=local_size))
+  prg = replace(prg, device=Device.DEFAULT)
+  if local_size is not None: prg = replace(prg, local_size=local_size)
+  ei = CompiledRunner(prg)
   ei.exec(outbufs+inbufs)
   return [np.frombuffer(x.as_buffer(), _to_np_dtype(x.dtype)) for x in outbufs]
 
@@ -37,8 +33,8 @@ def _setup_and_test_alu(alu_op:Ops, input_val:ConstType, *alu_src_uops:UOp):
   alu = ld.alu(alu_op, *alu_src_uops)
   store = UOp.store(a.index(idx), alu)
   sink = UOp(Ops.SINK, dtypes.void, (store,))
-  uops = full_rewrite(sink, Device[Device.DEFAULT].renderer)
-  return _test_uop_result([Tensor([input_val])], uops)[0]
+  prg = get_program(sink, Device[Device.DEFAULT].renderer)
+  return _test_uop_result([Tensor([input_val])], prg)[0]
 
 class TestRendererFailures(unittest.TestCase):
   @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer")
@@ -47,8 +43,8 @@ class TestRendererFailures(unittest.TestCase):
     gate_alu = (lidx0:=UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 4),), 'lidx0')).ne(0)
     gated_alu_store = UOp(Ops.STORE, dtypes.void, (a.index(lidx0.valid(gate_alu)), UOp.const(dtypes.int, 1)))
     sink = UOp(Ops.SINK, dtypes.void, (gated_alu_store,))
-    uops = full_rewrite(sink, Device[Device.DEFAULT].renderer)
-    ret = _test_uop_result([], uops, local_size=[4, 1, 1])[0]
+    prg = get_program(sink, Device[Device.DEFAULT].renderer)
+    ret = _test_uop_result([], prg, local_size=[4, 1, 1])[0]
     np.testing.assert_equal(ret, [0, 1, 1, 1])
 
   @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, (PTXRenderer, PythonRenderer)), "test is for ptx or python renderer")
@@ -58,8 +54,8 @@ class TestRendererFailures(unittest.TestCase):
     gate_alu_1 = (lidx1:=UOp(Ops.SPECIAL, dtypes.int, (UOp.const(dtypes.int, 2),), 'lidx1')).ne(0)
     gated_alu_store = UOp(Ops.STORE, dtypes.void, (a.index((lidx0+lidx1*4).valid(gate_alu_0&gate_alu_1)), UOp.const(dtypes.int, 1)))
     sink = UOp(Ops.SINK, dtypes.void, (gated_alu_store,))
-    uops = full_rewrite(sink, Device[Device.DEFAULT].renderer)
-    ret = _test_uop_result([], uops, local_size=[4, 2, 1])[0]
+    prg = get_program(sink, Device[Device.DEFAULT].renderer)
+    ret = _test_uop_result([], prg, local_size=[4, 2, 1])[0]
     np.testing.assert_equal(ret, [0, 0, 0, 0, 0, 1, 1, 1])
 
 @unittest.skipIf(not isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage), "uops are for cstyle")
@@ -104,8 +100,8 @@ class TestPTXFailures(unittest.TestCase):
     if_uop = UOp(Ops.IF, dtypes.void, (gate_alu,))
     gated_alu_store = UOp(Ops.STORE, dtypes.void, (a.index(lidx0, if_uop), val))
     sink = UOp(Ops.SINK, dtypes.void, (gated_alu_store,))
-    uops = full_rewrite(sink, Device[Device.DEFAULT].renderer)
-    ret = _test_uop_result([], uops, local_size=[4, 1, 1])[0]
+    prg = get_program(sink, Device[Device.DEFAULT].renderer)
+    ret = _test_uop_result([], prg, local_size=[4, 1, 1])[0]
     np.testing.assert_equal(ret, [0, 1, 1, 1])
 
   @unittest.skipUnless(is_dtype_supported(dtypes.half), "need half")

test/test_tensor.py

@@ -10,7 +10,7 @@ from tinygrad.device import is_dtype_supported
 from tinygrad.uop.ops import Ops, UOp
 from tinygrad.renderer.ptx import PTXRenderer
 from tinygrad.renderer.nir import NIRRenderer
-from tinygrad.codegen import full_rewrite
+from tinygrad.engine.realize import get_program
 from tinygrad.dtype import DType
 
 settings.register_profile("my_profile", max_examples=200, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False))
@@ -869,9 +869,8 @@ class TestIdxUpcast(unittest.TestCase):
     for s in schedule:
       if s.ast.op is Ops.SINK:
         renderer = Device[s.bufs[0].device].renderer
-        uops = full_rewrite(s.ast, renderer)
-        renderer.render(uops)
-        return uops
+        prg = get_program(s.ast, renderer)
+        return prg.uops
 
   def _assert(self, dtype: DType, a: Tensor):
     uops = self._schedule_render(a)

test/test_uops.py

@@ -7,30 +7,27 @@ from tinygrad.dtype import dtypes, DType, AddrSpace
 from tinygrad.device import Buffer, Device
 from tinygrad.uop.ops import Ops, UOp, UPat, KernelInfo, exec_alu, AxisType
 from tinygrad.uop.spec import shared_spec
-from tinygrad.renderer import ProgramSpec
 from tinygrad.renderer.cstyle import CStyleLanguage
 from tinygrad.engine.realize import CompiledRunner, get_program, get_runner
 from tinygrad.engine.schedule import ExecItem
-from tinygrad.codegen import full_rewrite
 from tinygrad.uop.symbolic import sym
 from tinygrad.device import is_dtype_supported
 from tinygrad.codegen.opt import Opt, OptOps
 from tinygrad.renderer.ptx import PTXRenderer
+from test.helpers import get_uops
+from dataclasses import replace
 
 def to_uops_list(u:list[UOp], ren=None) -> list[UOp]:
   sink = UOp.group(*u)
   for r in sink.ranges: sink = sink.end(r)
   # we strip the SINK here for legacy reasons
-  ret = full_rewrite(sink.sink(arg=KernelInfo(opts_to_apply=())), ren)
+  ret = get_uops(sink.sink(arg=KernelInfo(opts_to_apply=())), ren)
   assert ret[-1].op is Ops.SINK
   return ret[:-1]
 
 def _uops_to_prg(uops_list):
-  uops = full_rewrite(ast:=UOp.sink(*uops_list), ren=Device[Device.DEFAULT].renderer)
-  src = Device[Device.DEFAULT].renderer.render(uops)
-  has_local = Device[Device.DEFAULT].renderer.has_local
-  return CompiledRunner(ProgramSpec(uops[-1].arg.name if uops[-1].arg is not None else "test", src, Device.DEFAULT, ast, uops=uops,
-                                global_size=[1,1,1] if has_local else None, local_size=[1,1,1] if has_local else None))
+  prg = get_program(UOp.sink(*uops_list), Device[Device.DEFAULT].renderer)
+  return CompiledRunner(replace(prg, device=Device.DEFAULT))
 
 def uop(uops:list[UOp], uop:Ops, dtype:Optional[DType], src:tuple[UOp, ...], arg:Any=None) -> UOp:
   uops.append(UOp(uop, dtype, tuple(src), arg))
@@ -547,6 +544,12 @@ class TestUopsObject(unittest.TestCase):
     self.assertEqual(a.device, Device.DEFAULT)
 
 class TestUOpRender(unittest.TestCase):
+  def test_render_vectorize_empty(self):
+    u = UOp(Ops.VECTORIZE, dtype=dtypes.int.vec(0), src=())
+    self.assertEqual(u.render(simplify=False), "{}")
+  def test_render_vectorize_empty_simplified(self):
+    u = UOp(Ops.VECTORIZE, dtype=dtypes.int.vec(0), src=())
+    self.assertEqual(u.render(), "{}")
   def test_render_vectorize_same(self):
     u = UOp(Ops.VECTORIZE, dtype=dtypes.int.vec(3), src=(UOp.const(dtypes.int, 0), UOp.const(dtypes.int, 0), UOp.const(dtypes.int, 0)))
     self.assertEqual(u.render(simplify=False), "{0, ...}")

test/test_uops_stats.py

@@ -4,7 +4,6 @@ from tinygrad.helpers import getenv, GlobalCounters, EMULATE
 from tinygrad.engine.realize import get_program
 from tinygrad.renderer import ProgramSpec
 from tinygrad.renderer import Estimates
-from tinygrad.codegen import full_rewrite
 from tinygrad.uop.ops import Ops, UOp
 from tinygrad.dtype import dtypes
 from tinygrad.codegen.opt import Opt, OptOps, KernelOptError
@@ -146,7 +145,7 @@ class TestUOpsStats(unittest.TestCase):
     u3 = UOp(Ops.CONST, dtypes.int, tuple(), 3)
     u4 = UOp(Ops.MUL, dtypes.int, (u1,u2))
     u5 = UOp(Ops.ADD, dtypes.int, (u4,u3))
-    uops = full_rewrite(u5.sink())
+    uops = list(u5.toposort())
 
     globl = UOp(Ops.DEFINE_GLOBAL, dtypes.int.ptr(), tuple())
     o1 = UOp(Ops.CONST, dtypes.int, tuple(), 1)
@@ -155,7 +154,7 @@ class TestUOpsStats(unittest.TestCase):
     u2 = globl.index(o2)
     u3 = UOp(Ops.CONST, dtypes.int, tuple(), 3)
     u4 = UOp(Ops.MULACC, dtypes.int, (u1,u2,u3))
-    uops_fma = full_rewrite(u4.sink())
+    uops_fma = list(u4.toposort())
 
     self.assertEqual(flops_mem(uops), flops_mem(uops_fma))
 

test/testextra/test_cfg_viz.py

@@ -0,0 +1,180 @@
+# ruff: noqa: F405, F403
+# allow define from star imports
+
+import unittest
+import textwrap
+
+from tinygrad import Device, Tensor
+from tinygrad.uop.ops import UOp, Ops, track_rewrites
+from tinygrad.renderer import ProgramSpec
+from tinygrad.helpers import TracingKey, getenv
+from tinygrad.engine.realize import ExecItem, CompiledRunner
+
+from extra.assembly.amd.autogen.rdna3 import *
+
+# TODO: use the RDNA3 renderer when it's in master
+template = """.text
+.globl fn_name
+.p2align 8
+.type fn_name,@function
+fn_name:
+  INSTRUCTION
+
+.rodata
+.p2align 6
+.amdhsa_kernel fn_name
+  .amdhsa_kernarg_size 8
+  .amdhsa_user_sgpr_kernarg_segment_ptr 1
+  .amdhsa_next_free_vgpr .amdgcn.next_free_vgpr
+  .amdhsa_next_free_sgpr .amdgcn.next_free_sgpr
+  .amdhsa_wavefront_size32 1
+.end_amdhsa_kernel
+
+.amdgpu_metadata
+---
+amdhsa.version:
+  - 1
+  - 0
+amdhsa.kernels:
+  - .name: fn_name
+    .symbol: fn_name.kd
+    .group_segment_fixed_size: 0
+    .private_segment_fixed_size: 0
+    .wavefront_size: 32
+    .sgpr_count: 8
+    .vgpr_count: 8
+    .max_flat_workgroup_size: 1024
+    .kernarg_segment_align: 8
+    .kernarg_segment_size: 8
+    .args:
+      - .address_space:  global
+        .name:           a
+        .offset:         0
+        .size:           8
+        .type_name:      'float*'
+        .value_kind:     global_buffer
+...
+.end_amdgpu_metadata
+"""
+
+@track_rewrites(name=lambda *args,ret,**kwargs: TracingKey(ret.name, ret=ret))
+def run_asm(name:str, insts:list) -> ProgramSpec:
+  src = "\n".join([inst if isinstance(inst, str) else inst.disasm() for inst in insts])
+  prg = ProgramSpec(name, src:=template.replace("fn_name", name).replace("INSTRUCTION", textwrap.dedent(src)), Device.DEFAULT, UOp(Ops.SINK),
+                    lib=Device[Device.DEFAULT].compiler.compile(src), global_size=[1, 1, 1], local_size=[1, 1, 1], globals=[0])
+  ei = ExecItem(UOp(Ops.SINK), [Tensor.empty(1).uop.buffer.ensure_allocated()], prg=CompiledRunner(prg))
+  ei.run()
+  return prg
+
+@unittest.skipUnless(Device.DEFAULT == "AMD" and not getenv("AMD_LLVM"), "only on AMD with comgr")
+class TestCfg(unittest.TestCase):
+  def setUp(self):
+    arch = Device["AMD"].arch
+    if not any(arch.startswith(a) for a in {"gfx11", "gfx12"}):
+      self.skipTest(f"tests written for RDNA, got arch {arch}")
+
+  def test_simple(self):
+    run_asm("simple", [
+      "entry:",
+        s_branch("bb1"),
+      "bb1:",
+        s_endpgm(),
+    ])
+
+  def test_diamond(self):
+    run_asm("diamond", [
+      "entry:",
+        s_cmp_eq_i32(s[0], 0),
+        s_cbranch_scc1("if"),
+        s_branch("else"),
+      "if:",
+        s_nop(1),
+        s_branch("end"),
+      "else:",
+        s_nop(0),
+      "end:",
+        s_endpgm(),
+    ])
+
+  def test_loop(self):
+    run_asm("simple_loop", [
+      "entry:",
+        s_mov_b32(s[1], 4),
+      "loop:",
+        s_add_u32(s[1], s[1], -1),
+        s_cmp_eq_i32(s[1], 0),
+        s_cbranch_scc0("loop"),
+        s_endpgm(),
+    ])
+
+  def test_loop_branch(self):
+    run_asm("loop_if", [
+      "entry:",
+        s_mov_b32(s[1], 4),
+      "loop:",
+        s_add_u32(s[1], s[1], -1),
+        s_cmp_eq_i32(s[1], 2),
+        s_cbranch_scc1("cond"),
+        s_branch("cont"),
+      "cond:",
+        s_add_u32(s[1], s[1], -2),
+      "cont:",
+        s_cmp_eq_i32(s[1], 0),
+        s_cbranch_scc0("loop"),
+        s_endpgm(),
+    ])
+
+  def test_loop_break(self):
+    run_asm("loop_break", [
+      "entry:",
+        s_mov_b32(s[1], 8),
+      "loop:",
+        s_add_u32(s[1], s[1], -1),
+        s_cmp_eq_i32(s[1], 5),
+        s_cbranch_scc1("break"),
+        s_cmp_eq_i32(s[1], 0),
+        s_cbranch_scc0("loop"),
+      "break:",
+        s_endpgm(),
+    ])
+
+  def test_switch(self):
+    run_asm("switch_case", [
+      "entry:",
+        s_cmp_eq_i32(s[0], 0),
+        s_cbranch_scc1("case0"),
+        s_cmp_eq_i32(s[0], 1),
+        s_cbranch_scc1("case1"),
+        s_branch("case2"),
+      "case0:",
+        s_nop(0),
+        s_branch("join"),
+      "case1:",
+        s_nop(1),
+        s_branch("join"),
+      "case2:",
+        s_nop(2),
+        s_branch("join"),
+      "join:",
+        s_endpgm(),
+    ])
+
+  def test_ping_pong(self):
+    run_asm("ping_pong", [
+      "entry:",
+        s_cmp_eq_i32(s[0], 0),
+        s_cbranch_scc1("ping"),
+        s_branch("pong"),
+      "ping:",
+        s_cmp_eq_i32(s[1], 0),
+        s_cbranch_scc1("pong"),
+        s_branch("end"),
+      "pong:",
+        s_cmp_eq_i32(s[2], 0),
+        s_cbranch_scc1("ping"),
+      "end:",
+        s_endpgm(),
+    ])
+
+if __name__ == "__main__":
+  unittest.main()

test/unit/test_dtype.py

@@ -66,5 +66,21 @@ class TestDtypeTolist(unittest.TestCase):
     # 57344
     self.assertEqual(Tensor([-30000, 1.5, 3.1, 30000], device="PYTHON", dtype=dtypes.fp8e5m2).tolist(), [-28672.0, 1.5, 3.0, 28672.0])
 
+class TestCanLosslessCast(unittest.TestCase):
+  def test_can_lossless_cast(self):
+    from tinygrad.dtype import can_lossless_cast
+    # signed -> unsigned is NOT lossless (negative values wrap)
+    self.assertFalse(can_lossless_cast(dtypes.int8, dtypes.uint64))
+    self.assertFalse(can_lossless_cast(dtypes.int32, dtypes.uint32))
+    # unsigned -> larger signed is lossless
+    self.assertTrue(can_lossless_cast(dtypes.uint8, dtypes.int16))
+    self.assertTrue(can_lossless_cast(dtypes.uint32, dtypes.int64))
+    # large ints don't fit in floats
+    self.assertFalse(can_lossless_cast(dtypes.int32, dtypes.float))
+    self.assertFalse(can_lossless_cast(dtypes.int64, dtypes.double))
+    # half has more mantissa bits
+    self.assertTrue(can_lossless_cast(dtypes.int8, dtypes.half))
+    self.assertFalse(can_lossless_cast(dtypes.int8, dtypes.bfloat16))
+
 if __name__ == "__main__":
   unittest.main()

test/unit/test_dtype_spec.py

@@ -8,7 +8,7 @@ from hypothesis import given, settings, strategies as strat
 import numpy as np
 import torch
 
-settings.register_profile("my_profile", max_examples=200, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False))
+settings.register_profile("my_profile", max_examples=50, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False))
 settings.load_profile("my_profile")
 
 core_dtypes = list(DTYPES_DICT.values())

test/unit/test_elf.py

@@ -1,5 +1,5 @@
 import unittest, subprocess, platform
-from tinygrad.runtime.ops_cpu import ClangJITCompiler
+from tinygrad.runtime.support.compiler_cpu import ClangJITCompiler
 from tinygrad.runtime.support.elf import elf_loader
 
 class TestElfLoader(unittest.TestCase):

test/unit/test_hashing.py

@@ -47,10 +47,10 @@ class TestKeccak(unittest.TestCase):
 
       ha_ref, hb_ref = hasher(a), hasher(b)
       tres = Tensor.stack(*(Tensor(d) for d in (a, b))).keccak(name)
-      ha, hb = tres[0].data(), tres[1].data()
+      ha, hb = bytes(tres[0].data()), bytes(tres[1].data())
 
       self.assertEqual(ha_ref, ha)
-      self.assertEqual(ha_ref, Tensor(a).keccak(name).data())
+      self.assertEqual(ha_ref, bytes(Tensor(a).keccak(name).data()))
       self.assertEqual(hb_ref, hb)
 
   def test_referenced(self):

test/unit/test_helpers.py

@@ -1,6 +1,6 @@
-import ctypes, gzip, unittest, timeit
+import ctypes, gzip, unittest, timeit, pickle
 from tinygrad import Variable
-from tinygrad.helpers import Context, ContextVar, argfix, colored, word_wrap, is_numpy_ndarray, mv_address, get_contraction
+from tinygrad.helpers import Context, ContextVar, argfix, colored, word_wrap, is_numpy_ndarray, mv_address, get_contraction, count
 from tinygrad.helpers import merge_dicts, strip_parens, prod, round_up, fetch, fully_flatten, from_mv, to_mv, polyN, time_to_str, cdiv, cmod, getbits
 from tinygrad.tensor import Tensor, get_shape
 import numpy as np
@@ -120,6 +120,18 @@ class TestRoundUp(unittest.TestCase):
     self.assertEqual(round_up(232, 24984), 24984)
     self.assertEqual(round_up(24984, 232), 25056)
 
+class TestCount(unittest.TestCase):
+  def test_count_basic(self):
+    c = count(3)
+    self.assertEqual(next(c), 3)
+    self.assertEqual(next(c), 4)
+
+  def test_count_step_pickle(self):
+    c = count(1, 2)
+    self.assertEqual(next(c), 1)
+    c2 = pickle.loads(pickle.dumps(c))
+    self.assertEqual(next(c2), 3)
+
 @unittest.skip("no fetch tests because they need internet")
 class TestFetch(unittest.TestCase):
   def test_fetch_bad_http(self):

test/unit/test_llm_moe.py

@@ -0,0 +1,53 @@
+import unittest
+import numpy as np
+from tinygrad import Tensor
+
+class TestMoEFeedForward(unittest.TestCase):
+  def test_moe_feed_forward(self):
+    from tinygrad.apps.llm import TransformerBlock
+    dim, hidden, n_heads = 8, 16, 2
+    num_experts, k = 4, 2
+
+    block = TransformerBlock(dim, hidden, n_heads, n_heads, norm_eps=1e-5, head_dim=dim//n_heads,
+                             rope_theta=10000, max_context=16, num_experts=num_experts, num_experts_per_tok=k)
+
+    # set up weights: gate scales by (expert_id+1), up/down are identity-ish, router picks experts 0,2
+    block.ffn_gate_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) * (i + 1) for i in range(num_experts)])
+    block.ffn_up_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) for _ in range(num_experts)])
+    block.ffn_down_exps.weight = Tensor.stack(*[Tensor.eye(dim, hidden) for _ in range(num_experts)])
+    block.ffn_gate_inp.weight = Tensor([[1, 0, 1, 0]] * dim).T  # router strongly prefers experts 0 and 2
+    block.ffn_norm.weight = Tensor.ones(dim)  # identity norm
+
+    # input of ones -> after norm still ~ones -> experts 0,2 selected -> weighted sum of silu outputs
+    h = Tensor.ones(1, 1, dim)
+    out = block._feed_forward(h)
+
+    # expected: residual + moe_output ≈ 1 + avg(silu(1), silu(3))
+    expected = 1 + (Tensor([1.0]).silu().item() + Tensor([3.0]).silu().item()) / 2
+    np.testing.assert_allclose(out.numpy()[0, 0, 0], expected, rtol=1e-2)
+
+  def test_moe_feed_forward_batched(self):
+    from tinygrad.apps.llm import TransformerBlock
+    dim, hidden, n_heads = 8, 16, 2
+    num_experts, k = 4, 2
+
+    block = TransformerBlock(dim, hidden, n_heads, n_heads, norm_eps=1e-5, head_dim=dim//n_heads,
+                             rope_theta=10000, max_context=16, num_experts=num_experts, num_experts_per_tok=k)
+
+    # same setup as BS=1 test
+    block.ffn_gate_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) * (i + 1) for i in range(num_experts)])
+    block.ffn_up_exps.weight = Tensor.stack(*[Tensor.eye(hidden, dim) for _ in range(num_experts)])
+    block.ffn_down_exps.weight = Tensor.stack(*[Tensor.eye(dim, hidden) for _ in range(num_experts)])
+    block.ffn_gate_inp.weight = Tensor([[1, 0, 1, 0]] * dim).T
+    block.ffn_norm.weight = Tensor.ones(dim)
+
+    # test with BS=2, T=3
+    h = Tensor.ones(2, 3, dim)
+    out = block._feed_forward(h)
+
+    # all outputs should match the BS=1 expected value
+    expected = 1 + (Tensor([1.0]).silu().item() + Tensor([3.0]).silu().item()) / 2
+    np.testing.assert_allclose(out.numpy(), expected, rtol=1e-2)
+
+if __name__ == '__main__':
+  unittest.main()

test/unit/test_llm_server.py

@@ -27,8 +27,8 @@ class TestLLMServer(unittest.TestCase):
     from tinygrad.apps.llm import Handler
     from tinygrad.helpers import TCPServerWithReuse
 
-    cls.port = 11435
-    cls.server = TCPServerWithReuse(('127.0.0.1', cls.port), Handler)
+    cls.server = TCPServerWithReuse(('127.0.0.1', 0), Handler)
+    cls.port = cls.server.server_address[1]
     cls.server_thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
     cls.server_thread.start()
     time.sleep(0.1)

test/unit/test_tqdm.py

@@ -3,7 +3,7 @@ from unittest.mock import patch
 from io import StringIO
 from collections import namedtuple
 from tqdm import tqdm
-from tinygrad.helpers import tqdm as tinytqdm, trange as tinytrange
+from tinygrad.helpers import tqdm as tinytqdm, trange
 import numpy as np
 
 def _get_iter_per_second(raw:str) -> float:
@@ -165,7 +165,7 @@ class TestProgressBar(unittest.TestCase):
       mock_stderr.truncate(0)
 
       # compare bars at each iteration (only when tinytqdm bar has been updated)
-      for n in (bar := tinytrange(total, desc="Test")):
+      for n in (bar := trange(total, desc="Test")):
         if bar.i % bar.skip != 0: continue
         tiny_output = mock_stderr.getvalue().split("\r")[-1].rstrip()
         iters_per_sec = float(tiny_output.split("it/s")[-2].split(" ")[-1]) if n>0 else 0

test/unit/test_uop_repr.py

@@ -0,0 +1,38 @@
+import unittest
+from tinygrad import UOp, dtypes
+
+class TestUOpRepr(unittest.TestCase):
+  def test_simple_const(self):
+    a = UOp.const(dtypes.int, 42)
+    self.assertEqual(repr(a), "UOp(Ops.CONST, dtypes.int, arg=42, src=())")
+  def test_different_consts(self):
+    a, b = UOp.const(dtypes.int, 42), UOp.const(dtypes.int, 3)
+    expected = (
+      "UOp(Ops.ADD, dtypes.int, arg=None, src=(\n" +
+      "  UOp(Ops.CONST, dtypes.int, arg=42, src=()),\n" +
+      "  UOp(Ops.CONST, dtypes.int, arg=3, src=()),))"
+    )
+    self.assertEqual(repr(a+b), expected)
+  def test_walrus_operator_indentation(self):
+    # The reference should have the same indentation as the definition
+    a = UOp.const(dtypes.int, 42)
+    expected = (
+      "UOp(Ops.ADD, dtypes.int, arg=None, src=(\n" +
+      "  x0:=UOp(Ops.CONST, dtypes.int, arg=42, src=()),\n" +
+      "  x0,))"
+    )
+    self.assertEqual(repr(a+a), expected)
+  def test_nested_walrus_indentation(self):
+    # Ensure indentation is consistent at multiple levels
+    b = (a:=UOp.const(dtypes.int, 1)) + a
+    expected = (
+      "UOp(Ops.MUL, dtypes.int, arg=None, src=(\n" +
+      "  x0:=UOp(Ops.ADD, dtypes.int, arg=None, src=(\n" +
+      "    x1:=UOp(Ops.CONST, dtypes.int, arg=1, src=()),\n" +
+      "    x1,)),\n" +
+      "  x0,))"
+    )
+    self.assertEqual(repr(b*b), expected)
+
+if __name__ == '__main__':
+  unittest.main()