less sketch tests

.github/workflows/benchmark.yml

@@ -16,48 +16,6 @@ on:
   workflow_dispatch:
 
 jobs:
-  # the goal of this test is to replicate a normal person on a laptop running the test
-  # no process replay, no benchmarks, no CI, just a normal laptop person
-  # the 3 minute timeout should not be raised
-  testmacpytest:
-    name: Mac pytest
-    runs-on: [self-hosted, macOS]
-    timeout-minutes: 3
-    defaults:
-      run:
-        shell: bash -e -o pipefail {0}
-    if: github.repository_owner == 'tinygrad'
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v4
-    # brew install uv
-    - name: setup python environment
-      run: |
-        rm -rf /tmp/tinygrad_pytest_ci
-        uv venv /tmp/tinygrad_pytest_ci
-        source /tmp/tinygrad_pytest_ci/bin/activate
-        uv pip install .[testing]
-    - name: setup staging db
-      run: |
-        echo "CACHEDB=/tmp/pytest-db-ci.db" >> $GITHUB_ENV
-        rm -f /tmp/pytest-db-ci*
-    # TODO: remove this step once all old caches are migrated
-    - name: Migrate old huggingface cache (symlinks break onnxruntime 1.24+)
-      run: |
-        cd ~/Library/Caches/tinygrad/downloads/models 2>/dev/null || exit 0
-        for old_dir in models--*; do
-          [ -d "$old_dir" ] || continue
-          repo_id=$(echo "$old_dir" | sed 's/models--//; s/--/\//g')
-          snapshot=$(ls -1 "$old_dir/snapshots" 2>/dev/null | head -1)
-          [ -n "$snapshot" ] || continue
-          mkdir -p "$repo_id"
-          cp -RLn "$old_dir/snapshots/$snapshot/"* "$repo_id/" 2>/dev/null || true
-        done
-    - name: Run pytest -nauto
-      run: |
-        source /tmp/tinygrad_pytest_ci/bin/activate
-        pytest -nauto --durations=20
-
   testmacbenchmark:
     name: Mac Benchmark
     env:

.github/workflows/test.yml

@@ -29,9 +29,9 @@ jobs:
         deps: testing_unit
         llvm: 'true'
     - name: Speed Test
-      run: CPU=1 CPU_LLVM=1 THREADS=0 python3 test/speed/external_test_speed_v_torch.py
+      run: CPU=1 CPU_LLVM=1 python3 test/speed/external_test_speed_v_torch.py
     - name: Speed Test (BEAM=2)
-      run: BEAM=2 CPU=1 CPU_LLVM=1 THREADS=0 python3 test/speed/external_test_speed_v_torch.py
+      run: BEAM=2 CPU=1 CPU_LLVM=1 python3 test/speed/external_test_speed_v_torch.py
 
   docs:
     name: Docs
@@ -207,11 +207,11 @@ jobs:
       run: DEBUG=2 AMX=1 EMULATE=AMX FORWARD_ONLY=1 PYTHON=1 python3 test/opt/test_tensor_cores.py
     - name: Test device flop counts
       run: |
-        DEBUG=2 EMULATE=METAL PYTHON=1 python3 ./test/null/test_uops_stats.py TestUOpsStatsMatmulHalf
-        DEBUG=2 EMULATE=AMD PYTHON=1 python3 ./test/null/test_uops_stats.py TestUOpsStatsMatmulHalf
-        DEBUG=2 EMULATE=CUDA PYTHON=1 python3 ./test/null/test_uops_stats.py TestUOpsStatsMatmulHalf
-        DEBUG=2 EMULATE=INTEL PYTHON=1 python3 ./test/null/test_uops_stats.py TestUOpsStatsMatmulHalf
-        DEBUG=2 AMX=1 EMULATE=AMX PYTHON=1 python3 ./test/null/test_uops_stats.py TestUOpsStats.test_simple_matmul
+        DEBUG=2 EMULATE=METAL PYTHON=1 python3 ./test/test_uops_stats.py TestUOpsStatsMatmulHalf
+        DEBUG=2 EMULATE=AMD PYTHON=1 python3 ./test/test_uops_stats.py TestUOpsStatsMatmulHalf
+        DEBUG=2 EMULATE=CUDA PYTHON=1 python3 ./test/test_uops_stats.py TestUOpsStatsMatmulHalf
+        DEBUG=2 EMULATE=INTEL PYTHON=1 python3 ./test/test_uops_stats.py TestUOpsStatsMatmulHalf
+        DEBUG=2 AMX=1 EMULATE=AMX PYTHON=1 python3 ./test/test_uops_stats.py TestUOpsStats.test_simple_matmul
 
   linter:
     name: Linters
@@ -266,19 +266,17 @@ jobs:
       run: python -c "from tinygrad import Device; assert Device.DEFAULT == 'CPU', Device.DEFAULT"
     - name: Run unit tests
       run: |
-        CPU=1 python test/null/test_device.py TestRunAsModule.test_module_runs
-        CPU=1 python -m pytest -n=auto test/unit/ --durations=20
-    - name: Run NULL backend tests
-      run: NULL=1 python -m pytest -n=auto test/null/ --durations=20
+        CPU=1 python test/unit/test_device.py TestRunAsModule.test_module_runs
+        CPU=1 python -m pytest -n=auto test/unit/ --durations=20 --deselect=test/unit/test_device.py::TestRunAsModule::test_module_runs
     - name: Run targetted tests on NULL backend
-      run: NULL=1 python3 -m unittest test.test_multitensor.TestMultiTensor.test_data_parallel_resnet_train_step
+      run: NULL=1 python3 -m unittest test.test_multitensor.TestMultiTensor.test_data_parallel_resnet_train_step test/device/test_null.py
     # TODO: too slow
     # - name: Run SDXL on NULL backend
     #   run: NULL=1 DEBUG=1 python3 examples/sdxl.py --seed 0 --noshow --timing --fakeweights
     - name: Run Clip tests for SD MLPerf on NULL backend
       run: NULL=1 python -m pytest -n=auto test/external/mlperf_stable_diffusion/external_test_models.py::TestOpenClip --durations=20
     - name: Run AMD emulated BERT training on NULL backend
-      run: EMULATE=AMD_RDNA4 NULL=1 NULL_ALLOW_COPYOUT=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=1 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
+      run: EMULATE=AMD_RDNA4 NULL=1 CAPTURE_PROCESS_REPLAY=0 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=66 GPUS=1 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
     # TODO: support fake weights
     #- name: Run LLaMA 7B on 4 fake devices
     #  run: NULL=1 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 3 --temperature 0 --timing
@@ -316,7 +314,7 @@ jobs:
         deps: testing_unit
         python-version: '3.14'
     - name: Test SPEC=2
-      run: SPEC=2 pytest --maxfail=10 -n auto --durations=30 --ignore=test/models --ignore=test/null --ignore test/test_custom_kernel.py --ignore test/unit/test_hashing.py --timeout 60 -k "not test_setitem_big" --splits 2 --group ${{ matrix.group }}
+      run: SPEC=2 pytest --maxfail=10 -n auto --durations=30 --ignore=test/models --ignore test/test_custom_kernel.py --ignore test/unit/test_hashing.py --ignore test/unit/test_autogen.py --timeout 60 -k "not test_setitem_big" --splits 2 --group ${{ matrix.group }}
 
   fuzzing:
     name: Fuzzing
@@ -467,11 +465,11 @@ jobs:
       - name: Test MLPerf stuff
         run: CL=1 python -m pytest -n=auto test/external/external_test_optim.py test/external/external_test_losses.py test/external/external_test_metrics.py test/external/external_test_datasets.py --durations=20
       - name: NULL=1 beautiful_mnist_multigpu
-        run: NULL=1 NULL_ALLOW_COPYOUT=1 python examples/beautiful_mnist_multigpu.py
+        run: NULL=1 python examples/beautiful_mnist_multigpu.py
       - name: Test Bert training
-        run: NULL=1 NULL_ALLOW_COPYOUT=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=24 GPUS=4 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
+        run: NULL=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=24 GPUS=4 BERT_LAYERS=2 MODEL=bert python3 examples/mlperf/model_train.py
       - name: Test llama 3 training
-        run: NULL=1 NULL_ALLOW_COPYOUT=1 SAMPLES=300 BS=8 SEQLEN=512 GRADIENT_ACC_STEPS=1 FAKEDATA=1 DEFAULT_FLOAT=bfloat16 OPTIM_DTYPE=bfloat16 LLAMA3_SIZE=1B MODEL=llama3 python3 examples/mlperf/model_train.py
+        run: NULL=1 SAMPLES=300 BS=8 SEQLEN=512 GRADIENT_ACC_STEPS=1 FAKEDATA=1 DEFAULT_FLOAT=bfloat16 OPTIM_DTYPE=bfloat16 LLAMA3_SIZE=1B MODEL=llama3 python3 examples/mlperf/model_train.py
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
 
@@ -569,7 +567,7 @@ jobs:
       with:
         key: dsp-minimal
         deps: testing_unit
-        pydeps: "onnx==1.18.0 onnxruntime ml_dtypes"
+        pydeps: "onnx==1.18.0 onnxruntime"
         llvm: "true"
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
@@ -611,7 +609,7 @@ jobs:
         WEBGPU=1 DEBUG=4 FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
     - name: Run selected webgpu tests
       run: |
-          WEBGPU=1 WEBGPU_BACKEND="WGPUBackendType_Vulkan" python3 -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore=test/null --durations=20
+          WEBGPU=1 WEBGPU_BACKEND="WGPUBackendType_Vulkan" python3 -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --durations=20
     - name: Run process replay tests
       uses: ./.github/actions/process-replay
 
@@ -706,8 +704,6 @@ jobs:
       # TODO: run all once emulator is faster
       - name: Run RDNA3 ops tests
         run: SKIP_SLOW_TEST=1 AMD_LLVM=0 pytest -n=auto test/test_ops.py -k "test_sparse_categorical_crossentropy or test_tril or test_nonzero or test_softmax_argmax" --durations 20
-      - name: Run RDNA4 emulator tests
-        run: MOCKGPU_ARCH=rdna4 python -m pytest test/test_tiny.py -v --durations 20
 
   testnvidia:
     strategy:
@@ -739,7 +735,7 @@ jobs:
           DEBUG=5 FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
       - name: Run pytest (cuda)
         # skip multitensor because it's slow
-        run: python -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore=test/null --ignore test/test_gc.py --ignore test/test_multitensor.py --durations=20
+        run: python -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore test/test_gc.py --ignore test/test_multitensor.py --durations=20
       - name: Run TestOps.test_add with PMA
         run: VIZ=-1 PMA=1 DEBUG=5 python3 test/test_ops.py TestOps.test_add
       - name: Run process replay tests
@@ -772,7 +768,7 @@ jobs:
           python3 -c "from tinygrad import Device; assert Device.DEFAULT in ['CPU','CL'], Device.DEFAULT"
           DEBUG=5 FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
       - name: Run pytest (${{ matrix.backend }})
-        run: python -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore=test/null --durations=20
+        run: python -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --durations=20
       - name: Run TRANSCENDENTAL math
         run: TRANSCENDENTAL=2 python -m pytest -n=auto test/test_ops.py::TestOps::test_sin test/test_ops.py::TestOps::test_cos test/test_ops.py::TestOps::test_tan test/test_ops.py::TestOps::test_exp test/test_ops.py::TestOps::test_log --durations=20
       - name: Run process replay tests
@@ -799,8 +795,6 @@ jobs:
         llvm: 'true'
     - name: Run unit tests
       run: METAL=1 python -m pytest -n=auto test/unit/ --durations=20
-    - name: Run NULL backend tests
-      run: NULL=1 python -m pytest -n=auto test/null/ --durations=20
     - name: Run ONNX
       run: METAL=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
     - name: Test tensor core ops (fake)
@@ -902,7 +896,7 @@ jobs:
           python -c "from tinygrad import Device; assert Device.DEFAULT == {'LLVM':'CPU','LVP':'CPU'}.get(x:='${{ matrix.backend }}'.upper(), x), Device.DEFAULT"
           DEBUG=4 python3 test/test_tiny.py TestTiny.test_plus
       - name: Run pytest (${{ matrix.backend }})
-        run: python3 -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore=test/null --durations=20
+        run: python3 -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --durations=20
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
       - name: Run macOS-specific unit test
@@ -935,11 +929,7 @@ jobs:
       - name: Run unit tests
         if: matrix.backend=='llvm'
         # test_newton_schulz hits RecursionError
-        run: python -m pytest -n=auto test/unit/ --ignore=test/unit/test_disk_tensor.py --ignore=test/unit/test_tar.py --ignore=test/unit/test_linalg.py --durations=20
-      - name: Run NULL backend tests
-        if: matrix.backend=='llvm'
-        shell: bash
-        run: CPU=0 CPU_LLVM=0 NULL=1 python -m pytest -n=auto test/null/ --ignore=test/null/test_elf.py --durations=20
+        run: python -m pytest -n=auto test/unit/ --ignore=test/unit/test_disk_tensor.py --ignore=test/unit/test_elf.py --ignore=test/unit/test_tar.py --ignore=test/unit/test_linalg.py --durations=20
       - name: Run pytest (${{ matrix.backend }})
         shell: bash
         run: |
@@ -965,10 +955,10 @@ jobs:
           key: compile-${{ matrix.backend }}
           deps: testing_unit
           mesa: ${{ (matrix.backend == 'ir3' || matrix.backend == 'nak') && 'true' }}
-          python-version: '3.12'
+          python-version: '3.14'
       - name: Set env
         shell: bash
-        run: printf "NULL=1\nNULL_ALLOW_COPYOUT=1\n${{ matrix.backend == 'ir3' && 'NULL_IR3=1' || matrix.backend == 'nak' && 'NULL_NAK=1' }}" >> $GITHUB_ENV
+        run: printf "NULL=1\n${{ matrix.backend == 'ir3' && 'NULL_IR3=1' || matrix.backend == 'nak' && 'NULL_NAK=1' }}" >> $GITHUB_ENV
       - name: Run test_ops
         shell: bash
         run: |

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
         pass_filenames: false
       - id: tests
         name: comprehensive test suite
-        entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/test_ops.py test/test_schedule.py test/unit/test_assign.py test/test_tensor.py test/test_jit.py test/unit/test_schedule_cache.py test/null/test_pattern_matcher.py test/null/test_uop_symbolic.py test/unit/test_helpers.py
+        entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/test_ops.py test/test_schedule.py test/unit/test_assign.py test/test_tensor.py test/test_jit.py test/unit/test_schedule_cache.py test/unit/test_pattern_matcher.py test/unit/test_uop_symbolic.py test/unit/test_helpers.py
         language: system
         always_run: true
         pass_filenames: false

examples/anthropic_challenge.py

@@ -72,7 +72,7 @@ vliw_prepare = PatternMatcher([
   # cast is fake
   (UPat(Ops.CAST, name="c"), lambda c: c.src[0]),
   # rewrites to hardcode the addresses in memory
-  (UPat(Ops.PARAM, name="dg"), lambda dg: UOp.const(dtypes.uint, global_addrs[dg.arg])),
+  (UPat(Ops.DEFINE_GLOBAL, name="dg"), lambda dg: UOp.const(dtypes.uint, global_addrs[dg.arg])),
   # INDEX is just plus
   (UPat(Ops.INDEX, name="i"), lambda i: i.src[0]+i.src[1]),
 ])+symbolic

examples/mlperf/dataloader.py

@@ -72,7 +72,7 @@ def loader_process(q_in, q_out, X:Tensor, seed):
       #storage_tensor._copyin(img_tensor.numpy())
 
       # faster
-      X[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = img.tobytes()
+      X[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
 
       # ideal
       #X[idx].assign(img.tobytes())   # NOTE: this is slow!
@@ -264,8 +264,8 @@ def load_unet3d_data(preprocessed_dataset_dir, seed, queue_in, queue_out, X:Tens
       x = random_brightness_augmentation(x)
       x = gaussian_noise(x)
 
-    X[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = x.tobytes()
-    Y[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = y.tobytes()
+    X[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = x.tobytes()
+    Y[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = y.tobytes()
 
     queue_out.put(idx)
   queue_out.put(None)
@@ -379,12 +379,12 @@ def load_retinanet_data(base_dir:Path, val:bool, queue_in:Queue, queue_out:Queue
       clipped_match_idxs = np.clip(match_idxs, 0, None)
       clipped_boxes, clipped_labels = tgt["boxes"][clipped_match_idxs], tgt["labels"][clipped_match_idxs]
 
-      boxes[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = clipped_boxes.tobytes()
-      labels[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = clipped_labels.tobytes()
-      matches[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = match_idxs.tobytes()
-      anchors[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = anchor.tobytes()
+      boxes[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_boxes.tobytes()
+      labels[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = clipped_labels.tobytes()
+      matches[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = match_idxs.tobytes()
+      anchors[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = anchor.tobytes()
 
-    imgs[idx].contiguous().realize().uop.base.realized.as_memoryview(force_zero_copy=True)[:] = img.tobytes()
+    imgs[idx].contiguous().realize().uop.base.realized.as_buffer(force_zero_copy=True)[:] = img.tobytes()
 
     queue_out.put(idx)
   queue_out.put(None)

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_beam.sh

@@ -8,11 +8,9 @@ export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
 export DEBUG=${DEBUG:-2}
 export FLASH_ATTENTION=${FLASH_ATTENTION:-1}
 export ALL2ALL=${ALL2ALL:-1}
-export USE_ATOMICS=${USE_ATOMICS:-1}
-export ASM_GEMM=${ASM_GEMM:-1}
 
 export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=8 BS=16 EVAL_BS=8 GRADIENT_ACC_STEPS=1
+export DP=8 BS=8 EVAL_BS=8 GRADIENT_ACC_STEPS=2
 export GBS=$((BS * GRADIENT_ACC_STEPS))
 
 export MODEL="llama3"

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run.sh

@@ -10,10 +10,9 @@ export DEBUG=${DEBUG:-0}
 export FLASH_ATTENTION=${FLASH_ATTENTION:-1}
 export ALL2ALL=${ALL2ALL:-1}
 export USE_ATOMICS=${USE_ATOMICS:-1}
-export ASM_GEMM=${ASM_GEMM:-1}
 
 export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
-export DP=${DP:-8} BS=${BS:-16} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-1}
+export DP=${DP:-8} BS=${BS:-8} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2}
 export GBS=$((BS * GRADIENT_ACC_STEPS))
 
 export MODEL="llama3"

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/startup_walltime.sh

@@ -2,9 +2,8 @@
 export BENCHMARK=5
 export EVAL_BS=0
 export FAKEDATA=1
-export NULL_ALLOW_COPYOUT=1
 export HIP_VISIBLE_DEVICES=""
 export DEV=NULL
 export JITBEAM=0
 export LLAMA_LAYERS=${LLAMA_LAYERS:-"2"}
-time examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run.sh
+time examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run.sh

examples/stable_diffusion.py

@@ -6,6 +6,7 @@ import argparse, time
 from collections import namedtuple
 from typing import Dict, Any
 
+from PIL import Image
 import numpy as np
 from tinygrad import Device, GlobalCounters, dtypes, Tensor, TinyJit
 from tinygrad.helpers import Timing, Context, getenv, fetch, colored, tqdm, flatten, profile_marker
@@ -335,7 +336,6 @@ if __name__ == "__main__":
   print(x.shape)
 
   profile_marker("save image")
-  from PIL import Image
   im = Image.fromarray(x.numpy())
   print(f"saving {args.out}")
   im.save(args.out)

examples/tinychat/tinychat-browser/compile.py

@@ -48,7 +48,7 @@ def prepare_browser_chunks(model):
         weight_metadata = metadata.get(name, default)
         weight_metadata["parts"][part_num] = {"file": i, "file_start_pos": cursor, "size": size}
         metadata[name] = weight_metadata
-        data = bytes(state_dict[name].uop.base.realized.as_memoryview())
+        data = bytes(state_dict[name].uop.base.realized.as_buffer())
         data = data if not offsets else data[offsets[0]:offsets[1]]
         writer.write(data)
         cursor += size

extra/amdpci/proclogs.py

@@ -19,9 +19,8 @@ amdev = importlib.import_module("tinygrad.runtime.support.am.amdev")
 amdev.AMDev = AMDFake
 from tinygrad.runtime.ops_amd import PCIIface
 
-def parse_amdgpu_logs(log_content, register_names=None, register_objects=None, *, only_xcc0: bool = False):
+def parse_amdgpu_logs(log_content, register_names=None, *, only_xcc0: bool = False):
   register_map = register_names or {}
-  register_objs = register_objects or {}
 
   def replace_register(match):
     reg = match.group(1)
@@ -38,28 +37,6 @@ def parse_amdgpu_logs(log_content, register_names=None, register_objects=None, *
   # remove timing prefix
   processed_log = re.sub(r'^\[\s*\d+(?:\.\d+)?\]\s*', '', processed_log, flags=re.MULTILINE)
 
-  # decode register values into field dicts
-  def decode_value(match):
-    reg_name = match.group(1)
-    xcc_part = match.group(2)  # "xcc=0 " or ""
-    val_str = match.group(3)
-    val = int(val_str, 16)
-
-    reg_obj = register_objs.get(reg_name)
-    if reg_obj is not None and reg_obj.fields:
-      fields = reg_obj.decode(val)
-      # show raw for unaccounted bits
-      accounted = 0
-      for name, (start, end) in reg_obj.fields.items():
-        accounted |= (((1 << (end - start + 1)) - 1) << start)
-      unaccounted = val & ~accounted
-      parts = {k: v for k, v in fields.items() if v != 0}
-      if unaccounted: parts['_raw_unaccounted'] = hex(unaccounted)
-      return f"register {reg_name}, {xcc_part}with value {val_str} {parts}"
-    return match.group(0)
-
-  processed_log = re.sub(r'register (reg\w+), ((?:xcc=\d+ )?)with value (0x[0-9a-fA-F]+)', decode_value, processed_log)
-
   # keep only xcc=0 lines (but keep lines with no xcc at all)
   if only_xcc0:
     kept = []
@@ -73,18 +50,16 @@ def main():
   only_xcc0 = bool(getenv("ONLY_XCC0", 0))
 
   reg_names = {}
-  reg_objs = {}
   dev = PCIIface(None, 0)
   for x, y in dev.dev_impl.__dict__.items():
     if isinstance(y, AMRegister):
       for xcc, addr in y.addr.items():
         reg_names[addr] = f"{x}, xcc={xcc}"
-      reg_objs[x] = y
 
   with open(sys.argv[1], 'r') as f:
     log_content = f.read()
 
-  processed_log = parse_amdgpu_logs(log_content, reg_names, reg_objs, only_xcc0=only_xcc0)
+  processed_log = parse_amdgpu_logs(log_content, reg_names, only_xcc0=only_xcc0)
 
   with open(sys.argv[2], 'w') as f:
     f.write(processed_log)

extra/assembly/amd/emu.py

@@ -49,11 +49,10 @@ from tinygrad.helpers import Context, DEBUG, colored
 from tinygrad.engine.realize import get_runner
 
 from extra.assembly.amd import decode_inst
-from extra.assembly.amd.autogen.rdna3.str_pcode import PCODE as PCODE_RDNA3
-from extra.assembly.amd.autogen.rdna4.str_pcode import PCODE as PCODE_RDNA4
-from extra.assembly.amd.autogen.rdna3 import ins as ir3
-from extra.assembly.amd.autogen.rdna4 import ins as ir4
-from extra.assembly.amd.dsl import VCC_LO, EXEC_LO, SCC, ttmp
+from extra.assembly.amd.autogen.rdna3.str_pcode import PCODE
+from extra.assembly.amd.autogen.rdna3.ins import (SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, VOP1, VOP1_SDST, VOP2, VOP3, VOP3_SDST, VOP3SD, VOP3P, VOPC,
+  DS, FLAT, GLOBAL, SCRATCH, VOPD, SOPPOp, SMEMOp, VOP1Op, VOP2Op, VOP3Op, VOPDOp)
+from extra.assembly.amd.dsl import VCC_LO, EXEC_LO, SCC
 from extra.assembly.amd.autogen.common import Fmt, OpType
 from extra.assembly.amd.pcode import parse_block, _FUNCS
 
@@ -80,23 +79,15 @@ def _apply_src_mods(val: UOp, mod_bit: int, abs_bits: int, neg_bits: int, bits:
   if neg_bits & (1 << mod_bit): fv = fv.neg()
   return fv.bitcast(ut).cast(dtypes.uint32) if bits == 16 else fv.bitcast(ut)
 
-# Map VOPD ops to VOP2 ops for pcode lookup (both RDNA3 and RDNA4)
+# Map VOPD ops to VOP2 ops for pcode lookup
 VOPD_TO_VOP2 = {
-  ir3.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir3.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32,
-  ir3.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir3.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32,
-  ir3.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir3.VOPDOp.V_DUAL_MAX_F32: ir3.VOP2Op.V_MAX_F32_E32,
-  ir3.VOPDOp.V_DUAL_MIN_F32: ir3.VOP2Op.V_MIN_F32_E32, ir3.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32,
-  ir3.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir3.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32,
-  ir3.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir3.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32,
-  ir3.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32,
-  # RDNA4 mappings (same VOP1/VOP2 targets, RDNA4 uses _NUM_ suffix for min/max)
-  ir4.VOPDOp.V_DUAL_FMAC_F32: ir3.VOP2Op.V_FMAC_F32_E32, ir4.VOPDOp.V_DUAL_MUL_F32: ir3.VOP2Op.V_MUL_F32_E32,
-  ir4.VOPDOp.V_DUAL_ADD_F32: ir3.VOP2Op.V_ADD_F32_E32, ir4.VOPDOp.V_DUAL_SUB_F32: ir3.VOP2Op.V_SUB_F32_E32,
-  ir4.VOPDOp.V_DUAL_SUBREV_F32: ir3.VOP2Op.V_SUBREV_F32_E32, ir4.VOPDOp.V_DUAL_MAX_NUM_F32: ir3.VOP2Op.V_MAX_F32_E32,
-  ir4.VOPDOp.V_DUAL_MIN_NUM_F32: ir3.VOP2Op.V_MIN_F32_E32, ir4.VOPDOp.V_DUAL_ADD_NC_U32: ir3.VOP2Op.V_ADD_NC_U32_E32,
-  ir4.VOPDOp.V_DUAL_LSHLREV_B32: ir3.VOP2Op.V_LSHLREV_B32_E32, ir4.VOPDOp.V_DUAL_AND_B32: ir3.VOP2Op.V_AND_B32_E32,
-  ir4.VOPDOp.V_DUAL_MOV_B32: ir3.VOP1Op.V_MOV_B32_E32, ir4.VOPDOp.V_DUAL_CNDMASK_B32: ir3.VOP2Op.V_CNDMASK_B32_E32,
-  ir4.VOPDOp.V_DUAL_FMAAK_F32: ir3.VOP2Op.V_FMAAK_F32_E32, ir4.VOPDOp.V_DUAL_FMAMK_F32: ir3.VOP2Op.V_FMAMK_F32_E32,
+  VOPDOp.V_DUAL_FMAC_F32: VOP2Op.V_FMAC_F32_E32, VOPDOp.V_DUAL_MUL_F32: VOP2Op.V_MUL_F32_E32,
+  VOPDOp.V_DUAL_ADD_F32: VOP2Op.V_ADD_F32_E32, VOPDOp.V_DUAL_SUB_F32: VOP2Op.V_SUB_F32_E32,
+  VOPDOp.V_DUAL_SUBREV_F32: VOP2Op.V_SUBREV_F32_E32, VOPDOp.V_DUAL_MAX_F32: VOP2Op.V_MAX_F32_E32,
+  VOPDOp.V_DUAL_MIN_F32: VOP2Op.V_MIN_F32_E32, VOPDOp.V_DUAL_ADD_NC_U32: VOP2Op.V_ADD_NC_U32_E32,
+  VOPDOp.V_DUAL_LSHLREV_B32: VOP2Op.V_LSHLREV_B32_E32, VOPDOp.V_DUAL_AND_B32: VOP2Op.V_AND_B32_E32,
+  VOPDOp.V_DUAL_MOV_B32: VOP1Op.V_MOV_B32_E32, VOPDOp.V_DUAL_CNDMASK_B32: VOP2Op.V_CNDMASK_B32_E32,
+  VOPDOp.V_DUAL_FMAAK_F32: VOP2Op.V_FMAAK_F32_E32, VOPDOp.V_DUAL_FMAMK_F32: VOP2Op.V_FMAMK_F32_E32,
 }
 WAVE_SIZE = 32
 # Special registers stored after inline constants (256-259)
@@ -135,6 +126,14 @@ def _val_to_u32(val: UOp) -> UOp:
   if val.dtype in (dtypes.uint16, dtypes.int16): return val.cast(dtypes.uint32)
   return val.cast(dtypes.uint32)
 
+def _apply_clamp(val: UOp, clmp: int | UOp) -> UOp:
+  """Apply VOP3 clamp modifier: clamp float results to [0.0, 1.0] range."""
+  if isinstance(clmp, int) and clmp == 0: return val
+  if val.dtype not in (dtypes.float32, dtypes.half, dtypes.float64): return val
+  zero, one = UOp.const(val.dtype, 0.0), UOp.const(val.dtype, 1.0)
+  clamped = val.maximum(zero).minimum(one)
+  return clmp.ne(_c(0)).where(clamped, val) if isinstance(clmp, UOp) else clamped
+
 _pcode_fixes = {
   'V_DIV_FMAS_F32': ('D0.f32 = 2.0F ** 32 * fma(S0.f32, S1.f32, S2.f32)',
     'D0.f32 = (exponent(S2.f32) > 127) ? (2.0F ** 64 * fma(S0.f32, S1.f32, S2.f32)) : (2.0F ** -64 * fma(S0.f32, S1.f32, S2.f32))'),
@@ -147,15 +146,11 @@ _pcode_fixes = {
   'V_TRIG_PREOP_F64': ("result = 64'F((1201'B(2.0 / PI)[1200 : 0] << shift.u32) & 1201'0x1fffffffffffff)", "result = trig_preop_result(shift)"),
 }
 
-def _get_pcode_dict(op) -> dict:
-  """Return the PCODE dictionary for the given opcode based on its architecture."""
-  return PCODE_RDNA4 if 'rdna4' in type(op).__module__ else PCODE_RDNA3
-
 # Pcode parser
 @functools.cache
 def get_pcode(op) -> str:
   op_name = op.name
-  pcode = _get_pcode_dict(op)[op]
+  pcode = PCODE[op]
   if op_name in _pcode_fixes: pcode = pcode.replace(*_pcode_fixes[op_name])
   if 'V_DIV_SCALE' in op_name:
     dt, exp_lim, ldexp_val = ('f32', '23', '64') if 'F32' in op_name else ('f64', '52', '128')
@@ -179,12 +174,7 @@ def get_pcode(op) -> str:
 def parse_pcode(pcode: str, srcs: dict[str, UOp] | None = None) -> tuple[dict, list[tuple[str, UOp]]]:
   vars: dict = srcs.copy() if srcs else {}
   assigns: list[tuple[str, UOp]] = []
-  raw_lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')]
-  # TODO: pcode.py should tokenize full pcode string instead of line-by-line, then this hack can be removed
-  lines: list[str] = []
-  for l in raw_lines:
-    if lines and lines[-1].endswith('&&'): lines[-1] = lines[-1] + ' ' + l
-    else: lines.append(l)
+  lines = [l.strip().rstrip(';') for l in pcode.split('\n') if l.strip() and not l.strip().startswith('//')]
   _, final, _ = parse_block(lines, 0, vars, assigns=assigns)
   sliced = set(d.split('[')[0] for d, _ in assigns if '[' in d)
   for var, val in final.items():
@@ -257,11 +247,11 @@ def _collect_data_slices(assigns: list[tuple[str, UOp]], data_prefix: str, pcode
 class _Ctx:
   """Context for instruction compilation - holds buffers and helpers."""
   __slots__ = ('inst_size', 'dyn_fields', '_axis_id')
-  sgpr = UOp(Ops.PARAM, dtypes.uint32.ptr(SGPR_COUNT), arg=0)
-  vgpr = UOp(Ops.PARAM, dtypes.uint32.ptr(VGPR_SIZE), arg=1)
-  vmem = UOp(Ops.PARAM, dtypes.uint32.ptr(1 << 46), arg=2)
-  lds = UOp(Ops.PARAM, dtypes.uint32.ptr(16384), arg=3)
-  scratch = UOp(Ops.PARAM, dtypes.uint8.ptr(1 << 30), arg=4)
+  sgpr = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(SGPR_COUNT), arg=0)
+  vgpr = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(VGPR_SIZE), arg=1)
+  vmem = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(1 << 46), arg=2)
+  lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
+  scratch = UOp(Ops.DEFINE_GLOBAL, dtypes.uint8.ptr(1 << 30), arg=4)
 
   def __init__(self, inst_size: int):
     self.inst_size, self._axis_id = inst_size, 0
@@ -327,9 +317,9 @@ class _Ctx:
     return base, mask, size
 
   # Dynamic register access (takes UOp index instead of int)
-  def rsgpr_dyn(self, reg: UOp, valid: UOp | None = None) -> UOp:
+  def rsgpr_dyn(self, reg: UOp) -> UOp:
     """Read SGPR with dynamic register index."""
-    return self.sgpr.index(reg.cast(dtypes.int), valid, ptr=True).load() if valid is not None else self.sgpr.index(reg.cast(dtypes.int), ptr=True).load()
+    return self.sgpr.index(reg.cast(dtypes.int), ptr=True).load()
 
   def wsgpr_dyn(self, reg: UOp, val: UOp) -> UOp:
     """Write SGPR with dynamic register index. Writes to NULL (124) are discarded."""
@@ -346,23 +336,20 @@ class _Ctx:
     offset = reg.cast(dtypes.int) * _c(32, dtypes.int) + lane.cast(dtypes.int)
     return buf.index(offset, _lane_active(exec_mask, lane)).store(val.cast(dtypes.uint32))
 
-  def rsrc_dyn(self, off: UOp, lane: UOp | None, bits: int = 32, literal: UOp | None = None, is_f64: bool = False, do_cast: bool = True) -> UOp:
+  def rsrc_dyn(self, off: UOp, lane: UOp | None, bits: int = 32, literal: UOp | None = None, is_f64: bool = False) -> UOp:
     """Read source operand with dynamic offset. Handles SGPR/inline constants (<256), VGPR (>=256).
     If lane is None, only scalar access is supported (off must be < 256).
     is_f64: True for F64 operations where 64-bit literals go in high 32 bits."""
     is_float_const = (off >= _c(240)) & (off <= _c(248))
-    is_vgpr = off >= _c(256)
-    is_sgpr = is_vgpr.ne(True)
-    sgpr_lo = self.rsgpr_dyn(off, is_sgpr)
+    sgpr_lo = self.rsgpr_dyn(off)
 
     if lane is not None:
-      vgpr_reg = off - _c(256)
+      is_vgpr, vgpr_reg = off >= _c(256), off - _c(256)
       vgpr_lo = self.rvgpr_dyn(vgpr_reg, lane, is_vgpr)
       vgpr_val = _u64(vgpr_lo, self.rvgpr_dyn(vgpr_reg + _c(1), lane, is_vgpr)) if bits == 64 else vgpr_lo
 
     if bits == 64:
-      sgpr_hi = self.rsgpr_dyn(off + _c(1), is_sgpr)
-      sgpr_val = _u64(sgpr_lo, sgpr_hi)
+      sgpr_val = _u64(sgpr_lo, self.rsgpr_dyn(off + _c(1)))
       # Integer inline constants: sign-extend 32-bit value from buffer to 64-bit
       # Float constants: cast F32 to F64
       int_inline = sgpr_lo.cast(dtypes.int32).cast(dtypes.int64)
@@ -377,7 +364,7 @@ class _Ctx:
     else:
       scalar_val = sgpr_lo
       if literal is not None: scalar_val = off.eq(_c(255)).where(literal, scalar_val)
-      if bits == 16 and do_cast:  # Float constants: cast F32 to F16
+      if bits == 16:  # Float constants: cast F32 to F16
         scalar_val = is_float_const.where(scalar_val.bitcast(dtypes.float32).cast(dtypes.half).bitcast(dtypes.uint16).cast(dtypes.uint32), scalar_val)
 
     return is_vgpr.where(vgpr_val, scalar_val) if lane is not None else scalar_val
@@ -415,19 +402,17 @@ class _Ctx:
     return UOp.sink(*self.scalar_stores(assigns, sdst_reg, sdst_size), *self.inc_pc())
 
   def compile_lane_pcode(self, op, inst) -> UOp:
-    """Compile cross-lane ops (READLANE/WRITELANE/PERMLANE) using pcode parser."""
+    """Compile READLANE/READFIRSTLANE/WRITELANE using pcode parser."""
     pcode = get_pcode(op)
     op_name = op.name if hasattr(op, 'name') else str(op)
     src0_off, vdst_off = self.inst_field(type(inst).src0), self.inst_field(type(inst).vdst)
     src0_reg = (src0_off >= _c(256)).where(src0_off - _c(256), _c(0))  # VGPR index or 0
     src1_off = self.inst_field(type(inst).src1) if hasattr(type(inst), 'src1') else None
-    src2_off = self.inst_field(type(inst).src2) if hasattr(type(inst), 'src2') else None
     exec_lo = self.rsgpr_dyn(_c(EXEC_LO.offset))
     srcs = {
       'SRC0': src0_reg, 'VDST': vdst_off, 'EXEC_LO': exec_lo, 'EXEC': exec_lo.cast(dtypes.uint64), '_vgpr': self.vgpr,
       'S0': self.rsrc_dyn(src0_off, _c(0, dtypes.int)) if 'WRITELANE' in op_name else src0_reg,
       'S1': self.rsrc_dyn(src1_off, _c(0, dtypes.int)) if src1_off is not None else _c(0),
-      'S2': self.rsrc_dyn(src2_off, _c(0, dtypes.int)) if src2_off is not None else _c(0),
     }
     _, assigns = parse_pcode(pcode, srcs)
     stores = []
@@ -437,33 +422,14 @@ class _Ctx:
     return UOp.sink(*stores, *self.inc_pc())
 
   def compile_vop_pcode(self, op, srcs: dict[str, UOp], lane: UOp, vdst_reg: UOp, exec_mask: UOp,
-                        opsel_dst_hi: bool | UOp = False, sdst_reg: int | None = None, clmp: int = 0) -> UOp:
+                        opsel_dst_hi: bool | UOp = False, sdst_reg: int | None = None, clmp: int | UOp = 0) -> UOp:
     """Compile VOP instruction. Returns sink with stores and inc_pc."""
     pcode = get_pcode(op)
     vcc_reg = sdst_reg if sdst_reg is not None else VCC_LO.offset
     if 'VCC' not in srcs: srcs['VCC'] = self.rsgpr_dyn(_c(vcc_reg))
-    srcs.update({'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane,
-                 'ROUND_MODE': _c(0), 'ROUND_TOWARD_ZERO': _c(0)})  # rounding mode: 0=RNE, RTZ constant
+    srcs.update({'EXEC': exec_mask, 'SCC': self.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane})
     _, assigns = parse_pcode(pcode, srcs)
 
-    # For integer ops with clamp, compute overflow using wide arithmetic
-    # NOTE: MUL_LO ops don't saturate - they always return the low bits
-    int_saturate = None
-    if clmp and any(p in op.name for p in ('_NC_U', '_MAD_U', '_NC_I', '_MAD_I')):
-      is_signed, is_16bit = '_I' in op.name and '_U' not in op.name, '16' in op.name
-      if not (is_16bit and is_signed):  # Skip 16-bit signed ops due to codegen issues
-        s0, s1, s2 = srcs.get('S0'), srcs.get('S1'), srcs.get('S2')
-        if s0 is not None and s1 is not None:
-          narrow_dt = dtypes.uint16 if is_16bit else (dtypes.int32 if is_signed else dtypes.uint32)
-          wide_dt = dtypes.int32 if is_16bit else dtypes.int64
-          narrow_max, narrow_min = (0xFFFF, 0) if is_16bit else ((0x7FFFFFFF, -0x80000000) if is_signed else (0xFFFFFFFF, 0))
-          def to_wide(x): return (x.bitcast(narrow_dt) if x.dtype.itemsize == narrow_dt.itemsize else x.cast(narrow_dt)).cast(wide_dt)
-          is_sub, is_mad = 'SUB' in op.name, 'MAD' in op.name
-          full = (to_wide(s0) * to_wide(s1) + to_wide(s2)) if is_mad and s2 is not None else \
-                 (to_wide(s1) - to_wide(s0)) if is_sub and 'SUBREV' in op.name else \
-                 (to_wide(s0) - to_wide(s1)) if is_sub else (to_wide(s0) + to_wide(s1))
-          int_saturate = full.clamp(narrow_min, narrow_max).cast(narrow_dt)
-
     raw_stores: list = []
     vcc_val, exec_val = None, None
     for dest, val in assigns:
@@ -478,10 +444,7 @@ class _Ctx:
                        val.cast(dtypes.uint32) if val.dtype in (dtypes.uint16, dtypes.int16) else val.cast(dtypes.uint32) & UOp.const(dtypes.uint32, slice_mask)
             raw_stores.append(('vgpr_slice', (lo_bit, width, val_bits)))
             continue
-        # For integer ops with clamp, use pre-computed saturated value; for floats, clamp to [0,1]
-        if int_saturate is not None: val = int_saturate
-        elif clmp and val.dtype in (dtypes.float32, dtypes.half, dtypes.float64):
-          val = val.maximum(UOp.const(val.dtype, 0.0)).minimum(UOp.const(val.dtype, 1.0))
+        val = _apply_clamp(val, clmp)
         if val.dtype in (dtypes.uint64, dtypes.int64, dtypes.float64):
           lo, hi = _split64(val)
           raw_stores.extend([('vgpr', self.wvgpr_dyn(vdst_reg, lane, lo, exec_mask)), ('vgpr', self.wvgpr_dyn(vdst_reg + _c(1), lane, hi, exec_mask))])
@@ -516,14 +479,13 @@ class _Ctx:
 # INSTRUCTION HANDLERS
 # ═══════════════════════════════════════════════════════════════════════════════
 
-def _compile_sopp(inst: ir3.SOPP | ir4.SOPP, ctx: _Ctx) -> UOp:
-  simm16 = ctx.inst_field_signed(type(inst).simm16).cast(dtypes.int16)
-  if inst.op in (ir3.SOPPOp.S_ENDPGM, ir4.SOPPOp.S_ENDPGM):
+def _compile_sopp(inst: SOPP, ctx: _Ctx) -> UOp:
+  simm16 = ctx.inst_field_signed(SOPP.simm16).cast(dtypes.int16)
+  if inst.op == SOPPOp.S_ENDPGM:
     return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), UOp.const(dtypes.uint32, 0xFFFFFFFF)),
                           ctx.wsgpr_dyn(_c(PC_HI_IDX), UOp.const(dtypes.uint32, 0xFFFFFFFF)))
-  if inst.op in (ir3.SOPPOp.S_NOP, ir4.SOPPOp.S_NOP): return UOp.sink(*ctx.inc_pc())  # S_NOP is a no-op
   # NOTE: we ignore SOPPs without PCODE
-  if inst.op in _get_pcode_dict(inst.op):
+  if inst.op in PCODE:
     pcode = get_pcode(inst.op)
     pc_bytes = ctx.rpc()  # PC is already 64-bit byte address
     vcc, exec_lo = ctx.rsgpr_dyn(_c(VCC_LO.offset)), ctx.rsgpr_dyn(_c(EXEC_LO.offset))
@@ -535,57 +497,50 @@ def _compile_sopp(inst: ir3.SOPP | ir4.SOPP, ctx: _Ctx) -> UOp:
         return UOp.sink(ctx.wsgpr_dyn(_c(PC_LO_IDX), lo), ctx.wsgpr_dyn(_c(PC_HI_IDX), hi))
   return UOp.sink(*ctx.inc_pc())
 
-def _compile_smem(inst: ir3.SMEM | ir4.SMEM, ctx: _Ctx) -> UOp:
+def _compile_smem(inst: SMEM, ctx: _Ctx) -> UOp:
   # Cache invalidation instructions are no-ops in the emulator (we don't model caches)
-  cache_inv_ops = [ir3.SMEMOp.S_GL1_INV, ir3.SMEMOp.S_DCACHE_INV, ir4.SMEMOp.S_DCACHE_INV]
-  if hasattr(ir4.SMEMOp, 'S_GL1_INV'): cache_inv_ops.append(ir4.SMEMOp.S_GL1_INV)
-  if inst.op in cache_inv_ops:
+  if inst.op in (SMEMOp.S_GL1_INV, SMEMOp.S_DCACHE_INV):
     return UOp.sink(*ctx.inc_pc())
   # Dynamic sbase field (bits 5:0) - SGPR pair, field value * 2 = register offset
-  sbase = ctx.inst_field(type(inst).sbase) * _c(2)
+  sbase = ctx.inst_field(SMEM.sbase) * _c(2)
   # Dynamic sdata field (bits 12:6) - destination SGPR
-  sdata_reg = ctx.inst_field(type(inst).sdata)
-  # RDNA4 uses 'ioffset', RDNA3 uses 'offset' - use type(inst) to get correct field
-  offset_field = type(inst).ioffset if hasattr(type(inst), 'ioffset') else type(inst).offset
-  offset = ctx.inst_field_signed(offset_field)  # signed immediate
-  # Dynamic soffset field - SGPR for additional offset (NULL=124 reads as 0)
-  soffset = ctx.inst_field(type(inst).soffset)
+  sdata_reg = ctx.inst_field(SMEM.sdata)
+  offset = ctx.inst_field_signed(SMEM.offset)  # 21-bit signed immediate
+  # Dynamic soffset field (bits 63:57) - SGPR for additional offset (NULL=124 reads as 0)
+  soffset = ctx.inst_field(SMEM.soffset)
   addr = _u64(ctx.rsgpr_dyn(sbase), ctx.rsgpr_dyn(sbase + _c(1))) + offset.cast(dtypes.uint64) + ctx.rsgpr_dyn(soffset).cast(dtypes.uint64)
-  _SMEM_NDWORDS = {ir3.SMEMOp.S_LOAD_B32: 1, ir3.SMEMOp.S_LOAD_B64: 2, ir3.SMEMOp.S_LOAD_B128: 4,
-    ir3.SMEMOp.S_LOAD_B256: 8, ir3.SMEMOp.S_LOAD_B512: 16, ir4.SMEMOp.S_LOAD_B32: 1, ir4.SMEMOp.S_LOAD_B64: 2,
-    ir4.SMEMOp.S_LOAD_B96: 3, ir4.SMEMOp.S_LOAD_B128: 4, ir4.SMEMOp.S_LOAD_B256: 8, ir4.SMEMOp.S_LOAD_B512: 16}
-  ndwords = _SMEM_NDWORDS[inst.op]
+  ndwords = {SMEMOp.S_LOAD_B32: 1, SMEMOp.S_LOAD_B64: 2, SMEMOp.S_LOAD_B128: 4, SMEMOp.S_LOAD_B256: 8, SMEMOp.S_LOAD_B512: 16}.get(inst.op, 1)
   stores = [ctx.wsgpr_dyn(sdata_reg + _c(i), ctx.vmem.index((addr + UOp.const(dtypes.uint64, i * 4) >> UOp.const(dtypes.uint64, 2)).cast(dtypes.int)))
             for i in range(ndwords)]
   return UOp.sink(*stores, *ctx.inc_pc())
 
-def _compile_sop(inst: ir3.SOP1 | ir3.SOP2 | ir3.SOPC | ir3.SOPK | ir4.SOP1 | ir4.SOP2 | ir4.SOPC | ir4.SOPK, ctx: _Ctx) -> UOp:
+def _compile_sop(inst: SOP1 | SOP2 | SOPC | SOPK, ctx: _Ctx) -> UOp:
   bits = inst.canonical_op_bits
   literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
 
-  if isinstance(inst, (ir3.SOPK, ir4.SOPK)):
-    sdst_off = ctx.inst_field(type(inst).sdst)
-    simm16 = ctx.inst_field(type(inst).simm16)
+  if isinstance(inst, SOPK):
+    sdst_off = ctx.inst_field(SOPK.sdst)
+    simm16 = ctx.inst_field(SOPK.simm16)
     # Sign-extend simm16
     simm16_sext = simm16.cast(dtypes.int16).cast(dtypes.int32)
     srcs = {'S0': ctx.rsgpr_dyn(sdst_off), 'SIMM16': simm16_sext, 'D0': ctx.rsgpr_dyn(sdst_off)}
     dst_off, dst_size = sdst_off, 1
-  elif isinstance(inst, (ir3.SOP1, ir4.SOP1)):
-    sdst_off = ctx.inst_field(type(inst).sdst)
-    ssrc0_off = ctx.inst_field(type(inst).ssrc0)
+  elif isinstance(inst, SOP1):
+    sdst_off = ctx.inst_field(SOP1.sdst)
+    ssrc0_off = ctx.inst_field(SOP1.ssrc0)
     srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal)}
     dst_off, dst_size = sdst_off, bits['d'] // 32
-  elif isinstance(inst, (ir3.SOP2, ir4.SOP2)):
-    sdst_off = ctx.inst_field(type(inst).sdst)
-    ssrc0_off = ctx.inst_field(type(inst).ssrc0)
-    ssrc1_off = ctx.inst_field(type(inst).ssrc1)
+  elif isinstance(inst, SOP2):
+    sdst_off = ctx.inst_field(SOP2.sdst)
+    ssrc0_off = ctx.inst_field(SOP2.ssrc0)
+    ssrc1_off = ctx.inst_field(SOP2.ssrc1)
     srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal),
             'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)}
     if literal is not None: srcs['SIMM32'] = literal
     dst_off, dst_size = sdst_off, bits['d'] // 32
-  elif isinstance(inst, (ir3.SOPC, ir4.SOPC)):
-    ssrc0_off = ctx.inst_field(type(inst).ssrc0)
-    ssrc1_off = ctx.inst_field(type(inst).ssrc1)
+  elif isinstance(inst, SOPC):
+    ssrc0_off = ctx.inst_field(SOPC.ssrc0)
+    ssrc1_off = ctx.inst_field(SOPC.ssrc1)
     srcs = {'S0': ctx.rsrc_dyn(ssrc0_off, None, bits['s0'], literal),
             'S1': ctx.rsrc_dyn(ssrc1_off, None, bits['s1'], literal)}
     dst_off, dst_size = _c(0), 0  # SOPC writes to SCC, not sdst
@@ -594,18 +549,18 @@ def _compile_sop(inst: ir3.SOP1 | ir3.SOP2 | ir3.SOPC | ir3.SOPK | ir4.SOP1 | ir
 
   return ctx.compile_sop_pcode(inst.op, srcs, dst_off, dst_size)
 
-def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP2 | ir4.VOP1 | ir4.VOP1_SDST | ir4.VOP2, ctx: _Ctx) -> UOp:
+def _compile_vop12(inst: VOP1 | VOP1_SDST | VOP2, ctx: _Ctx) -> UOp:
   op_name = _op_name(inst)
-  if op_name in ('V_READFIRSTLANE_B32_E32', 'V_PERMLANE64_B32_E32'): return ctx.compile_lane_pcode(inst.op, inst)
+  if op_name == 'V_READFIRSTLANE_B32_E32': return ctx.compile_lane_pcode(inst.op, inst)
   lane, exec_mask, bits = ctx.range(), ctx.rsgpr_dyn(_c(EXEC_LO.offset)), inst.canonical_op_bits
   literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
-  vdst_reg = ctx.inst_field(type(inst).vdst)
+  vdst_reg = ctx.inst_field(VOP1.vdst)
   write_hi_half = bits['d'] == 16 and (vdst_reg >= _c(128))
   if isinstance(write_hi_half, UOp): vdst_reg = write_hi_half.where(vdst_reg - _c(128), vdst_reg)
   elif write_hi_half: vdst_reg -= 128
-  if isinstance(inst, (ir3.VOP1, ir4.VOP1)):
+  if isinstance(inst, VOP1):
     # Handle VOP1 hi-half source operand (src0 >= v[128] for 16-bit ops)
-    src0_off = ctx.inst_field(type(inst).src0)
+    src0_off = ctx.inst_field(VOP1.src0)
     s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal)
     if bits['s0'] == 16:
       src0_hi = src0_off >= _c(384)
@@ -614,13 +569,13 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP2 | ir4.VOP1 | ir4.VO
       s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0)
     srcs = {'S0': s0}
   else:
-    vsrc1_reg = ctx.inst_field(type(inst).vsrc1)
+    vsrc1_reg = ctx.inst_field(VOP2.vsrc1)
     vsrc1_hi = bits['s0'] == 16 and (vsrc1_reg >= _c(128))
     vsrc1_actual = _cond(vsrc1_hi, vsrc1_reg - _c(128), vsrc1_reg)
     s1 = _cond_hi16(vsrc1_hi, ctx.rvgpr_dyn(vsrc1_actual, lane))
     d0 = _cond_hi16(write_hi_half, ctx.rvgpr_dyn(vdst_reg, lane))  # FMAC/FMAMK hi-half dest needs hi-half accumulator
     # Handle VOP2 hi-half src0 operand (src0 >= v[128] for 16-bit ops)
-    src0_off = ctx.inst_field(type(inst).src0)
+    src0_off = ctx.inst_field(VOP2.src0)
     s0 = ctx.rsrc_dyn(src0_off, lane, bits['s0'], literal)
     if bits['s0'] == 16:
       src0_hi = src0_off >= _c(384)
@@ -628,20 +583,19 @@ def _compile_vop12(inst: ir3.VOP1 | ir3.VOP1_SDST | ir3.VOP2 | ir4.VOP1 | ir4.VO
       src0_reg = src0_hi.where(src0_off - _c(384), _c(0))
       s0 = src0_hi.where(_hi16(ctx.rvgpr_dyn(src0_reg, lane)), s0)
     srcs = {'S0': s0, 'S1': s1, 'D0': d0}
-    if inst.op in (ir3.VOP2Op.V_FMAAK_F32_E32, ir3.VOP2Op.V_FMAMK_F32_E32, ir3.VOP2Op.V_FMAAK_F16_E32,
-                   ir3.VOP2Op.V_FMAMK_F16_E32):
+    if inst.op in (VOP2Op.V_FMAAK_F32_E32, VOP2Op.V_FMAMK_F32_E32, VOP2Op.V_FMAAK_F16_E32, VOP2Op.V_FMAMK_F16_E32):
       assert literal is not None
       srcs['SIMM32'] = literal
   return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=write_hi_half)
 
-def _compile_vopc(inst: ir3.VOPC | ir3.VOP3 | ir4.VOPC | ir4.VOP3, ctx: _Ctx, opsel: int = 0, abs_bits: int = 0, neg_bits: int = 0) -> UOp:
+def _compile_vopc(inst: VOPC | VOP3, ctx: _Ctx, opsel: int = 0, abs_bits: int = 0, neg_bits: int = 0) -> UOp:
   exec_mask, op_name, bits = ctx.rsgpr_dyn(_c(EXEC_LO.offset)), _op_name(inst), inst.canonical_op_bits
   is_cmpx, is_vopc = 'CMPX' in op_name, hasattr(inst, 'vsrc1')  # is_vopc: e32 vs e64
 
   # Handle both VOPC (vsrc1) and VOP3 (src1) instruction formats - read operands dynamically
   if is_vopc:
-    src0_off = ctx.inst_field(type(inst).src0)
-    vsrc1_off = ctx.inst_field(type(inst).vsrc1)
+    src0_off = ctx.inst_field(VOPC.src0)
+    vsrc1_off = ctx.inst_field(VOPC.vsrc1)
     # For 16-bit ops, vsrc1 >= 128 means hi-half of v[vsrc1-128]
     if bits['s0'] == 16:
       vsrc1_hi = vsrc1_off >= _c(128)
@@ -650,9 +604,9 @@ def _compile_vopc(inst: ir3.VOPC | ir3.VOP3 | ir4.VOPC | ir4.VOP3, ctx: _Ctx, op
       vsrc1_hi = False
       src1_off = _c(256) + vsrc1_off
   else:
-    src0_off = ctx.inst_field(type(inst).src0)
-    src1_off = ctx.inst_field(type(inst).src1)
-    dst_off = ctx.inst_field(type(inst).vdst)
+    src0_off = ctx.inst_field(VOP3.src0)
+    src1_off = ctx.inst_field(VOP3.src1)
+    dst_off = ctx.inst_field(VOP3.vdst)
     vsrc1_hi = False
   literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
 
@@ -681,7 +635,7 @@ def _compile_vopc(inst: ir3.VOPC | ir3.VOP3 | ir4.VOPC | ir4.VOP3, ctx: _Ctx, op
     stores = [ctx.wsgpr_dyn(dst_off, new_result)] if not is_vopc else [ctx.wsgpr_dyn(_c(VCC_LO.offset), new_result)]
   return UOp.sink(*stores, *ctx.inc_pc())
 
-def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3, ctx: _Ctx) -> UOp:
+def _compile_vop3(inst: VOP3, ctx: _Ctx) -> UOp:
   exec_mask = ctx.rsgpr_dyn(_c(EXEC_LO.offset))
   bits = inst.canonical_op_bits
   opsel, op_name = getattr(inst, 'opsel', 0) or 0, _op_name(inst)
@@ -690,22 +644,18 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3, ctx: _Ctx) -> UOp:
   if op_name in ('V_READLANE_B32', 'V_READFIRSTLANE_B32', 'V_READFIRSTLANE_B32_E64', 'V_WRITELANE_B32'):
     return ctx.compile_lane_pcode(inst.op, inst)
 
-  # V_PERMLANE16_B32 / V_PERMLANEX16_B32: cross-lane swizzle via pcode
-  if 'PERMLANE16' in op_name or 'PERMLANEX16' in op_name:
-    return ctx.compile_lane_pcode(inst.op, inst)
-
   # VOP3 VOPC (v_cmp_*_e64) - delegate to unified VOPC handler
   if 'V_CMP' in op_name or 'V_CMPX' in op_name:
     return _compile_vopc(inst, ctx, opsel=opsel, abs_bits=getattr(inst, 'abs', 0) or 0, neg_bits=getattr(inst, 'neg', 0) or 0)
 
   # Regular VOP3 - read operands dynamically
   lane = ctx.range()
-  vdst_reg = ctx.inst_field(type(inst).vdst)
+  vdst_reg = ctx.inst_field(VOP3.vdst)
   literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
   ops = inst.canonical_operands
-  src0 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), lane, bits['s0'], literal, 's0' in ops and ops['s0'][0] == Fmt.FMT_NUM_F64)
-  src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, bits['s1'], literal, 's1' in ops and ops['s1'][0] == Fmt.FMT_NUM_F64)
-  src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, bits['s2'], literal, 's2' in ops and ops['s2'][0] == Fmt.FMT_NUM_F64)
+  src0 = ctx.rsrc_dyn(ctx.inst_field(VOP3.src0), lane, bits['s0'], literal, 's0' in ops and ops['s0'][0] == Fmt.FMT_NUM_F64)
+  src1 = ctx.rsrc_dyn(ctx.inst_field(VOP3.src1), lane, bits['s1'], literal, 's1' in ops and ops['s1'][0] == Fmt.FMT_NUM_F64)
+  src2 = ctx.rsrc_dyn(ctx.inst_field(VOP3.src2), lane, bits['s2'], literal, 's2' in ops and ops['s2'][0] == Fmt.FMT_NUM_F64)
   if bits['s0'] == 16:
     src0 = _apply_opsel(src0, 0, opsel)
     src1 = _apply_opsel(src1, 1, opsel)
@@ -715,19 +665,19 @@ def _compile_vop3(inst: ir3.VOP3 | ir4.VOP3, ctx: _Ctx) -> UOp:
   src1 = _apply_src_mods(src1, 1, abs_bits, neg_bits, bits['s1'])
   src2 = _apply_src_mods(src2, 2, abs_bits, neg_bits, bits['s2'])
   srcs = {'S0': src0, 'S1': src1, 'S2': src2}
-  if inst.op in (ir3.VOP3Op.V_CNDMASK_B32_E64, ir3.VOP3Op.V_CNDMASK_B16) and src2 is not None: srcs['VCC'] = src2
+  if inst.op in (VOP3Op.V_CNDMASK_B32_E64, VOP3Op.V_CNDMASK_B16) and src2 is not None: srcs['VCC'] = src2
   # FMAC instructions need D0 (accumulator) from destination register
   if 'FMAC' in op_name: srcs['D0'] = ctx.rvgpr_dyn(vdst_reg, lane)
   opsel_dst_hi = bool(opsel & 0b1000) and bits['d'] == 16
   return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, opsel_dst_hi=opsel_dst_hi, clmp=getattr(inst, 'clmp', 0))
 
-def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD, ctx: _Ctx) -> UOp:
+def _compile_vop3sd(inst: VOP3SD, ctx: _Ctx) -> UOp:
   exec_mask = ctx.rsgpr_dyn(_c(EXEC_LO.offset))
   bits, pcode, ops = inst.canonical_op_bits, get_pcode(inst.op), inst.canonical_operands
 
   # Read operands dynamically from instruction encoding
-  vdst_reg, sdst_off = ctx.inst_field(type(inst).vdst), ctx.inst_field(type(inst).sdst)
-  src0_off, src1_off, src2_off = ctx.inst_field(type(inst).src0), ctx.inst_field(type(inst).src1), ctx.inst_field(type(inst).src2)
+  vdst_reg, sdst_off = ctx.inst_field(VOP3SD.vdst), ctx.inst_field(VOP3SD.sdst)
+  src0_off, src1_off, src2_off = ctx.inst_field(VOP3SD.src0), ctx.inst_field(VOP3SD.src1), ctx.inst_field(VOP3SD.src2)
   literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
 
   has_carry_in = 's2' in ops and ops['s2'][2] == OpType.OPR_SREG
@@ -745,7 +695,6 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD, ctx: _Ctx) -> UOp:
   _, assigns = parse_pcode(pcode, srcs)
 
   has_per_lane_vcc = any('[laneId]' in dest for dest, _ in assigns if dest.startswith('VCC') or dest.startswith('D0.u64'))
-  clmp = getattr(inst, 'clmp', 0)
   if has_per_lane_vcc:
     # VCC computation: RANGE+REDUCE gets axis ID first (lower ID = runs first)
     # This ensures VCC reads source values BEFORE VGPR stores modify them
@@ -757,17 +706,11 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD, ctx: _Ctx) -> UOp:
     final_vcc = ctx.unroll_lanes(get_vcc_bit, exec_mask)
     # VGPR stores: RANGE gets axis ID second (higher ID = runs after VCC loop)
     lane3 = ctx.range()
-    d0_val, vcc_per_lane = None, None
+    d0_val = None
     for dest, val in parse_pcode(pcode, load_srcs(lane3))[1]:
       if dest.startswith('D0') and '[laneId]' not in dest: d0_val = val
-      if dest.startswith('VCC') or (dest.startswith('D0.u64') and '[laneId]' in dest): vcc_per_lane = val
     vgpr_stores = []
     if d0_val is not None:
-      # Apply clamp using carry/borrow bit: ADD overflow->0xFFFFFFFF, SUB underflow->0
-      if clmp and vcc_per_lane is not None:
-        is_sub = 'SUB' in inst.op.name
-        sat_val = _c(0) if is_sub else _c(0xFFFFFFFF)
-        d0_val = vcc_per_lane.cast(dtypes.bool).where(sat_val, d0_val.cast(dtypes.uint32))
       if d0_val.dtype in (dtypes.uint64, dtypes.int64, dtypes.float64):
         lo, hi = _split64(d0_val)
         vgpr_stores.extend([ctx.wvgpr_dyn(vdst_reg, lane3, lo, exec_mask), ctx.wvgpr_dyn(vdst_reg + _c(1), lane3, hi, exec_mask)])
@@ -780,13 +723,13 @@ def _compile_vop3sd(inst: ir3.VOP3SD | ir4.VOP3SD, ctx: _Ctx) -> UOp:
   else:
     return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask, sdst_reg=inst.sdst.offset)
 
-def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P, ctx: _Ctx) -> UOp:
+def _compile_wmma(inst: VOP3P, ctx: _Ctx) -> UOp:
   op_name = _op_name(inst)
   exec_mask = ctx.rsgpr_dyn(_c(EXEC_LO.offset))
-  vdst_reg = ctx.inst_field(type(inst).vdst)
-  src0_r = ctx.inst_field(type(inst).src0) - _c(256)
-  src1_r = ctx.inst_field(type(inst).src1) - _c(256)
-  src2_r = ctx.inst_field(type(inst).src2) - _c(256)
+  vdst_reg = ctx.inst_field(VOP3P.vdst)
+  src0_r = ctx.inst_field(VOP3P.src0) - _c(256)
+  src1_r = ctx.inst_field(VOP3P.src1) - _c(256)
+  src2_r = ctx.inst_field(VOP3P.src2) - _c(256)
   is_f16_output = 'F16_16X16X16_F16' in op_name or 'BF16_16X16X16_BF16' in op_name  # F16/BF16 output vs F32 output
   is_bf16 = 'BF16' in op_name
   cvt = _FUNCS['bf16_to_f32'] if is_bf16 else _FUNCS['f16_to_f32']
@@ -813,17 +756,16 @@ def _compile_wmma(inst: ir3.VOP3P | ir4.VOP3P, ctx: _Ctx) -> UOp:
     stores = [ctx.wvgpr_dyn(vdst_reg + _c(i // 32), UOp.const(dtypes.int, i % 32), mat_d[i].bitcast(dtypes.uint32), exec_mask) for i in range(256)]
   return UOp.sink(*stores, *ctx.inc_pc())
 
-def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P, ctx: _Ctx) -> UOp:
+def _compile_vop3p(inst: VOP3P, ctx: _Ctx) -> UOp:
   op_name = _op_name(inst)
   if 'WMMA' in op_name and ('16X16X16_F16' in op_name or '16X16X16_BF16' in op_name): return _compile_wmma(inst, ctx)
 
   lane = ctx.range()
   exec_mask = ctx.rsgpr_dyn(_c(EXEC_LO.offset))
-  vdst_reg = ctx.inst_field(type(inst).vdst)
-  do_cast = any(x in op_name for x in ('F16', 'F32', 'BF16')) and 'IU' not in op_name
-  src0 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src0), lane, 16, do_cast=do_cast)
-  src1 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src1), lane, 16, do_cast=do_cast)
-  src2 = ctx.rsrc_dyn(ctx.inst_field(type(inst).src2), lane, 16, do_cast=do_cast)
+  vdst_reg = ctx.inst_field(VOP3P.vdst)
+  src0 = ctx.rsrc_dyn(ctx.inst_field(VOP3P.src0), lane, 16)
+  src1 = ctx.rsrc_dyn(ctx.inst_field(VOP3P.src1), lane, 16)
+  src2 = ctx.rsrc_dyn(ctx.inst_field(VOP3P.src2), lane, 16)
   opsel, opsel_hi = getattr(inst, 'opsel', 0) or 0, getattr(inst, 'opsel_hi', 3) if getattr(inst, 'opsel_hi', 3) is not None else 3
   opsel_hi2 = getattr(inst, 'opsel_hi2', 1) if getattr(inst, 'opsel_hi2', 1) is not None else 1
   neg, neg_hi = getattr(inst, 'neg', 0) or 0, getattr(inst, 'neg_hi', 0) or 0
@@ -845,7 +787,7 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P, ctx: _Ctx) -> UOp:
     s0_mod = apply_neg_mix(apply_abs(src0, 1, 1, 1), 1, 1, 1)
     s1_mod = apply_neg_mix(apply_abs(src1, 2, 2, 2), 2, 2, 2)
     s2_mod = apply_neg_mix(apply_abs(src2, 4, 4, 4), 4, 4, 4)
-    srcs = {'S@0': s0_mod, 'S@1': s1_mod, 'S@2': s2_mod,
+    srcs = {'S0': s0_mod, 'S1': s1_mod, 'S2': s2_mod,
             'OPSEL_HI': UOp.const(dtypes.uint32, combined_opsel_hi), 'OPSEL': UOp.const(dtypes.uint32, opsel)}
   else:
     def get_half_bits(val: UOp, use_hi: bool, apply_neg: bool = False) -> UOp:
@@ -854,28 +796,24 @@ def _compile_vop3p(inst: ir3.VOP3P | ir4.VOP3P, ctx: _Ctx) -> UOp:
       return bits
     def build_remapped_src(src: UOp, opsel_lo_bit: int, opsel_hi_bit: int, neg_lo_bit: int, neg_hi_bit: int) -> UOp:
       return get_half_bits(src, bool(opsel_lo_bit), bool(neg_lo_bit)) | (get_half_bits(src, bool(opsel_hi_bit), bool(neg_hi_bit)) << UOp.const(dtypes.uint32, 16))
-    # DOT IU instructions use NEG bits for signed/unsigned selection, not fp16 negation
-    is_dot_iu = 'DOT' in op_name and 'IU' in op_name
-    n0, n1, n2, nh0, nh1, nh2 = (0, 0, 0, 0, 0, 0) if is_dot_iu else (neg & 1, neg & 2, neg & 4, neg_hi & 1, neg_hi & 2, neg_hi & 4)
-    srcs = {'S0': build_remapped_src(src0, opsel & 1, opsel_hi & 1, n0, nh0),
-            'S1': build_remapped_src(src1, opsel & 2, opsel_hi & 2, n1, nh1),
-            'S2': build_remapped_src(src2, opsel & 4, 1 if opsel_hi2 else 0, n2, nh2)}
-    if is_dot_iu: srcs['NEG'] = UOp.const(dtypes.uint32, neg)
+    s0_new = build_remapped_src(src0, opsel & 1, opsel_hi & 1, neg & 1, neg_hi & 1)
+    s1_new = build_remapped_src(src1, opsel & 2, opsel_hi & 2, neg & 2, neg_hi & 2)
+    s2_new = build_remapped_src(src2, opsel & 4, 1 if opsel_hi2 else 0, neg & 4, neg_hi & 4)
+    srcs = {'S0': s0_new, 'S1': s1_new, 'S2': s2_new}
   return ctx.compile_vop_pcode(inst.op, srcs, lane, vdst_reg, exec_mask)
 
-def _compile_vopd(inst: ir3.VOPD | ir4.VOPD, ctx: _Ctx) -> UOp:
+def _compile_vopd(inst: VOPD, ctx: _Ctx) -> UOp:
   exec_mask = ctx.rsgpr_dyn(_c(EXEC_LO.offset))
-  # Read operands dynamically - use type(inst) to get correct field descriptors
-  inst_type = type(inst)
-  vdstx_reg = ctx.inst_field(inst_type.vdstx)
+  # Read operands dynamically
+  vdstx_reg = ctx.inst_field(VOPD.vdstx)
   # vdsty has complex encoding: actual = (raw << 1) | ((vdstx & 1) ^ 1)
-  vdsty_raw = ctx.inst_field(inst_type.vdsty)
+  vdsty_raw = ctx.inst_field(VOPD.vdsty)
   vdsty_reg = (vdsty_raw << _c(1)) | ((vdstx_reg & _c(1)) ^ _c(1))
-  srcx0_off = ctx.inst_field(inst_type.srcx0)
-  srcy0_off = ctx.inst_field(inst_type.srcy0)
-  vsrcx1_reg = ctx.inst_field(inst_type.vsrcx1)
-  vsrcy1_reg = ctx.inst_field(inst_type.vsrcy1)
-  literal = ctx.inst_field(inst_type.literal) if hasattr(inst_type, 'literal') else None
+  srcx0_off = ctx.inst_field(VOPD.srcx0)
+  srcy0_off = ctx.inst_field(VOPD.srcy0)
+  vsrcx1_reg = ctx.inst_field(VOPD.vsrcx1)
+  vsrcy1_reg = ctx.inst_field(VOPD.vsrcy1)
+  literal = ctx.inst_field(type(inst).literal) if hasattr(type(inst), 'literal') else None
 
   lane = ctx.range()
   srcy0, srcy1 = ctx.rsrc_dyn(srcy0_off, lane, literal=literal), ctx.rvgpr_dyn(vsrcy1_reg, lane)
@@ -886,64 +824,49 @@ def _compile_vopd(inst: ir3.VOPD | ir4.VOPD, ctx: _Ctx) -> UOp:
     assert vop is not None, f"no VOP mapping for VOPD {label}: {op}"
     if label == 'Y': srcs = {'S0': srcy0, 'S1': srcy1, 'D0': ctx.rvgpr_dyn(vdst_reg, lane)}
     else: srcs = {'S0': ctx.rsrc_dyn(src0_off, lane, literal=literal), 'S1': ctx.rvgpr_dyn(vsrc1_reg, lane), 'D0': ctx.rvgpr_dyn(vdst_reg, lane)}
-    if op in (ir3.VOPDOp.V_DUAL_FMAAK_F32, ir3.VOPDOp.V_DUAL_FMAMK_F32, ir4.VOPDOp.V_DUAL_FMAAK_F32, ir4.VOPDOp.V_DUAL_FMAMK_F32):
+    if op in (VOPDOp.V_DUAL_FMAAK_F32, VOPDOp.V_DUAL_FMAMK_F32):
       assert literal is not None
       srcs['SIMM32'] = literal
-    if op in (ir3.VOPDOp.V_DUAL_CNDMASK_B32, ir4.VOPDOp.V_DUAL_CNDMASK_B32): srcs['VCC'] = ctx.rsgpr_dyn(_c(VCC_LO.offset))
+    if op == VOPDOp.V_DUAL_CNDMASK_B32: srcs['VCC'] = ctx.rsgpr_dyn(_c(VCC_LO.offset))
     pcode = get_pcode(vop)
     srcs.update({'VCC': ctx.rsgpr_dyn(_c(VCC_LO.offset)), 'EXEC': exec_mask, 'SCC': ctx.rsgpr_dyn(_c(SCC.offset)), 'laneId': lane})
     for dest, val in parse_pcode(pcode, srcs)[1]:
       if dest.startswith('D0'): all_stores.append(ctx.wvgpr_dyn(vdst_reg, lane, _val_to_u32(val), exec_mask, after=srcy1))
   return UOp.sink(UOp.group(*all_stores).end(lane), *ctx.inc_pc())
 
-def _compile_mem_op(inst: ir3.DS | ir3.FLAT | ir3.GLOBAL | ir3.SCRATCH | ir4.DS | ir4.VFLAT | ir4.VGLOBAL | ir4.VSCRATCH, ctx: _Ctx) -> UOp:
+def _compile_mem_op(inst: DS | FLAT | GLOBAL | SCRATCH, ctx: _Ctx) -> UOp:
   """Unified memory operation compiler for DS, FLAT, GLOBAL, SCRATCH."""
   exec_mask, op_name = ctx.rsgpr_dyn(_c(EXEC_LO.offset)), _op_name(inst)
   pcode = get_pcode(inst.op)
 
-  is_lds = isinstance(inst, (ir3.DS, ir4.DS))
-  is_scratch = isinstance(inst, (ir3.SCRATCH, ir4.VSCRATCH))
+  is_lds = isinstance(inst, DS)
+  is_scratch = isinstance(inst, SCRATCH)
   mem = ctx.lds if is_lds else ctx.scratch if is_scratch else ctx.vmem
   addr_shift = UOp.const(dtypes.uint32 if is_lds else dtypes.uint64, 2)
 
   # Extract register info - all dynamic for deduplication
   if is_lds:
-    addr_reg = ctx.inst_field(type(inst).addr)
-    vdata_reg = ctx.inst_field(type(inst).data0)
-    vdst_reg = ctx.inst_field(type(inst).vdst)
-    offset0 = ctx.inst_field(type(inst).offset0)
-    offset1 = ctx.inst_field(type(inst).offset1)
+    addr_reg = ctx.inst_field(DS.addr)
+    vdata_reg = ctx.inst_field(DS.data0)
+    vdst_reg = ctx.inst_field(DS.vdst)
+    offset0 = ctx.inst_field(DS.offset0)
+    offset1 = ctx.inst_field(DS.offset1)
     offset = offset0  # DS uses offset0 as primary offset
     saddr_reg = None
-  elif isinstance(inst, (ir4.VGLOBAL, ir4.VSCRATCH, ir4.VFLAT)):  # RDNA4: vaddr, vsrc, ioffset
-    addr_reg = ctx.inst_field(type(inst).vaddr)
-    vdata_reg = ctx.inst_field(type(inst).vsrc)
-    vdst_reg = ctx.inst_field(type(inst).vdst)
-    offset = ctx.inst_field_signed(type(inst).ioffset)
-    offset0, offset1 = _c(0), _c(0)
-    saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None
-  else:  # RDNA3: addr, data, offset
+  else:
     addr_reg = ctx.inst_field(type(inst).addr)
     vdata_reg = ctx.inst_field(type(inst).data)
     vdst_reg = ctx.inst_field(type(inst).vdst)
     offset = ctx.inst_field_signed(type(inst).offset)
     offset0, offset1 = _c(0), _c(0)
-    saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(type(inst), 'saddr') else None
+    # Dynamic saddr - read field, NULL (124) or >= 128 means no saddr
+    saddr_reg = ctx.inst_field(type(inst).saddr) if hasattr(inst, 'saddr') else None
 
   # Data width from canonical_op_bits (32/64/96/128), default to 32 for untyped ops
   data_bits_mem = inst.canonical_op_bits.get('data', 32)
   is_atomic, glc = 'ATOMIC' in op_name, getattr(inst, 'glc', 0)
   has_data1 = is_lds and hasattr(inst, 'data1') and inst.data1 is not None
-  data1_reg = ctx.inst_field(type(inst).data1) if is_lds else _c(0)
-
-  # DS_PERMUTE/DS_BPERMUTE: cross-lane VGPR access via pcode
-  if is_lds and 'PERMUTE' in op_name:
-    pcode = get_pcode(inst.op)
-    srcs = {'ADDR': addr_reg, 'DATA0': vdata_reg, 'VDST': vdst_reg, 'OFFSET': offset,
-            'EXEC': exec_mask.cast(dtypes.uint64), '_vgpr': ctx.vgpr}
-    _, assigns = parse_pcode(pcode, srcs)
-    stores = [ctx.vgpr.index(val[0].cast(dtypes.int)).store(val[1].cast(dtypes.uint32)) for dest, val in assigns if dest.startswith('VGPR[')]
-    return UOp.sink(*stores, *ctx.inc_pc())
+  data1_reg = ctx.inst_field(DS.data1) if is_lds else _c(0)
 
   def make_addr(lane: UOp) -> UOp:
     if is_lds: return ctx.rvgpr_dyn(addr_reg, lane)
@@ -985,26 +908,14 @@ def _compile_mem_op(inst: ir3.DS | ir3.FLAT | ir3.GLOBAL | ir3.SCRATCH | ir4.DS
       else:
         data = {'DATA': _u64(ctx.rvgpr_dyn(vdata_reg, lane), ctx.rvgpr_dyn(vdata_reg + _c(1), lane)),
                 'DATA2': _u64(ctx.rvgpr_dyn(data1_reg, lane), ctx.rvgpr_dyn(data1_reg + _c(1), lane)) if has_data1 else UOp.const(dtypes.uint64, 0)}
-      # RDNA3 uses ADDR/OFFSET, RDNA4 uses vgpr_a/offset (lowercase) + CalcDsAddr function
-      return {'ADDR': addr, 'ADDR_BASE': addr, 'OFFSET': offset, 'OFFSET0': offset0, 'OFFSET1': offset1, '_lds': mem, 'laneId': lane,
-              'vgpr_a': ctx.rvgpr_dyn(addr_reg, lane), 'offset': offset, **data}
+      return {'ADDR': addr, 'ADDR_BASE': addr, 'OFFSET': offset, 'OFFSET0': offset0, 'OFFSET1': offset1, '_lds': mem, 'laneId': lane, **data}
     active = _lane_active(exec_mask, lane)
-    # saddr < 124 means valid SGPR pair, otherwise use 0 (NULL means no saddr contribution)
-    use_saddr = (saddr_reg < _c(124)) if saddr_reg is not None else UOp.const(dtypes.bool, False)
-    saddr_raw = _u64(ctx.rsgpr_dyn(saddr_reg), ctx.rsgpr_dyn(saddr_reg + _c(1))) if saddr_reg is not None else UOp.const(dtypes.uint64, 0)
-    saddr_base = use_saddr.where(saddr_raw, UOp.const(dtypes.uint64, 0))
-    # Sign-extend offset to 64-bit for the final address calculation
-    ioffset64 = offset.cast(dtypes.int64).cast(dtypes.uint64)
-    # v_addr for CalcGlobalAddr: when saddr valid, use low 32 bits as offset; otherwise full 64-bit address. Include ioffset.
-    vaddr_full = _u64(ctx.rvgpr_dyn(addr_reg, lane), ctx.rvgpr_dyn(addr_reg + _c(1), lane))
-    vaddr_lo = ctx.rvgpr_dyn(addr_reg, lane).cast(dtypes.uint64)
-    vaddr_base = use_saddr.where(vaddr_lo + ioffset64, vaddr_full + ioffset64)
     if is_atomic:
       return {'ADDR': addr, 'DATA': _u64(ctx.rvgpr_dyn(vdata_reg, lane), ctx.rvgpr_dyn(vdata_reg + _c(1), lane)) if data_bits_mem == 64 else ctx.rvgpr_dyn(vdata_reg, lane),
-              '_vmem': mem, '_active': active, 'laneId': lane, 'v_addr': vaddr_base, 's_saddr': saddr_base}
+              '_vmem': mem, '_active': active, 'laneId': lane}
     vdata = ctx.rvgpr_dyn(vdata_reg, lane).cast(dtypes.uint64) if 'STORE' in op_name else ctx.rvgpr_dyn(vdst_reg, lane) if 'D16' in op_name else UOp.const(dtypes.uint32, 0)
     if 'STORE' in op_name and data_bits_mem >= 64: vdata = vdata | (ctx.rvgpr_dyn(vdata_reg + _c(1), lane).cast(dtypes.uint64) << UOp.const(dtypes.uint64, 32))
-    srcs = {'ADDR': addr, 'VDATA': vdata, '_vmem': mem, '_active': active, 'laneId': lane, 'v_addr': vaddr_base, 's_saddr': saddr_base}
+    srcs = {'ADDR': addr, 'VDATA': vdata, '_vmem': mem, '_active': active, 'laneId': lane}
     for i in range(data_bits_mem // 32): srcs[f'VDATA{i}'] = ctx.rvgpr_dyn(vdata_reg + _c(i), lane) if 'STORE' in op_name else UOp.const(dtypes.uint32, 0)
     return srcs
 
@@ -1055,15 +966,10 @@ def _compile_mem_op(inst: ir3.DS | ir3.FLAT | ir3.GLOBAL | ir3.SCRATCH | ir4.DS
 
 # Dispatch table: instruction type -> handler function
 _INST_HANDLERS: dict[type, Callable[..., UOp]] = {
-  ir3.SOPP: _compile_sopp, ir3.SMEM: _compile_smem, ir3.SOP1: _compile_sop, ir3.SOP2: _compile_sop, ir3.SOPC: _compile_sop, ir3.SOPK: _compile_sop,
-  ir3.VOP1: _compile_vop12, ir3.VOP1_SDST: _compile_vop12, ir3.VOP2: _compile_vop12, ir3.VOPC: _compile_vopc, ir3.VOP3: _compile_vop3,
-  ir3.VOP3_SDST: _compile_vop3, ir3.VOP3SD: _compile_vop3sd, ir3.VOP3P: _compile_vop3p, ir3.VOPD: _compile_vopd,
-  ir3.DS: _compile_mem_op, ir3.FLAT: _compile_mem_op, ir3.GLOBAL: _compile_mem_op, ir3.SCRATCH: _compile_mem_op,
-  # RDNA4 instruction classes
-  ir4.SOPP: _compile_sopp, ir4.SMEM: _compile_smem, ir4.SOP1: _compile_sop, ir4.SOP2: _compile_sop, ir4.SOPC: _compile_sop, ir4.SOPK: _compile_sop,
-  ir4.VOP1: _compile_vop12, ir4.VOP1_SDST: _compile_vop12, ir4.VOP2: _compile_vop12, ir4.VOPC: _compile_vopc, ir4.VOP3: _compile_vop3,
-  ir4.VOP3_SDST: _compile_vop3, ir4.VOP3SD: _compile_vop3sd, ir4.VOP3P: _compile_vop3p, ir4.VOPD: _compile_vopd,
-  ir4.DS: _compile_mem_op, ir4.VFLAT: _compile_mem_op, ir4.VGLOBAL: _compile_mem_op, ir4.VSCRATCH: _compile_mem_op,
+  SOPP: _compile_sopp, SMEM: _compile_smem, SOP1: _compile_sop, SOP2: _compile_sop, SOPC: _compile_sop, SOPK: _compile_sop,
+  VOP1: _compile_vop12, VOP1_SDST: _compile_vop12, VOP2: _compile_vop12, VOPC: _compile_vopc, VOP3: _compile_vop3, VOP3_SDST: _compile_vop3,
+  VOP3SD: _compile_vop3sd, VOP3P: _compile_vop3p, VOPD: _compile_vopd,
+  DS: _compile_mem_op, FLAT: _compile_mem_op, GLOBAL: _compile_mem_op, SCRATCH: _compile_mem_op,
 }
 
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -1073,9 +979,9 @@ _INST_HANDLERS: dict[type, Callable[..., UOp]] = {
 _canonical_runner_cache: list[tuple[int, int, int, object]] = []  # [(base, mask, size, runner), ...]
 
 @functools.cache
-def _get_runner(inst_bytes: bytes, arch: str = "rdna3"):
+def _get_runner(inst_bytes: bytes):
   """Build and compile instruction to CompiledRunner. Cached by instruction bytes, with canonical dedup."""
-  inst = decode_inst(inst_bytes, arch)
+  inst = decode_inst(inst_bytes)
   inst_size = inst.size()
   inst_int = int.from_bytes(inst_bytes[:inst_size], 'little')
 
@@ -1104,15 +1010,15 @@ def _get_runner(inst_bytes: bytes, arch: str = "rdna3"):
   return runner, True
 
 @functools.cache
-def decode_program(data: bytes, arch: str = "rdna3") -> dict[int, tuple[str, Callable, list[int], Any]]:
+def decode_program(data: bytes) -> dict[int, tuple[str, Callable, list[int], Any]]:
   """Decode program to {pc: (name, fxn, globals, runner)}."""
   result: dict[int, tuple[str, Callable, list[int], Any]] = {}
   i = 0
   while i < len(data):
-    inst = decode_inst(data[i:], arch)
-    if hasattr(inst, 'op') and inst.op in (ir3.SOPPOp.S_CODE_END, ir4.SOPPOp.S_CODE_END): break
+    inst = decode_inst(data[i:])
+    if isinstance(inst, SOPP) and inst.op == SOPPOp.S_CODE_END: break
     try:
-      runner, is_new = _get_runner(bytes(data[i:i + inst.size() + 4]), arch)
+      runner, is_new = _get_runner(bytes(data[i:i + inst.size() + 4]))
       if DEBUG >= 3:
         try: inst_str = repr(inst)
         except Exception: inst_str = f"<{type(inst).__name__} at PC={i}>"
@@ -1141,8 +1047,8 @@ class WaveState:
     self.n_lanes = n_lanes
     self.vgpr_buf = Buffer('CPU', VGPR_SIZE, dtypes.uint32).ensure_allocated()
     self.sgpr_buf = Buffer('CPU', SGPR_COUNT, dtypes.uint32).ensure_allocated()
-    self._vgpr_mv = self.vgpr_buf.as_memoryview(force_zero_copy=True).cast('I')
-    self._sgpr_mv = self.sgpr_buf.as_memoryview(force_zero_copy=True).cast('I')
+    self._vgpr_mv = self.vgpr_buf.as_buffer(force_zero_copy=True).cast('I')
+    self._sgpr_mv = self.sgpr_buf.as_buffer(force_zero_copy=True).cast('I')
     # Zero memory using ctypes memset (much faster than Python loops)
     ctypes.memset(self.vgpr_buf._buf.va_addr, 0, VGPR_SIZE * 4)
     ctypes.memset(self.sgpr_buf._buf.va_addr, 0, SGPR_COUNT * 4)
@@ -1171,9 +1077,9 @@ class WaveState:
 # ═══════════════════════════════════════════════════════════════════════════════
 
 def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int, rsrc2: int = 0x19c,
-            scratch_size: int = 0, arch: str = "rdna3") -> int:
+            scratch_size: int = 0) -> int:
   """Execute AMD assembly program. scratch_size is private_segment_fixed_size from kernel descriptor (per-lane)."""
-  program_raw = decode_program(bytes((ctypes.c_char * lib_sz).from_address(lib).raw), arch)
+  program_raw = decode_program(bytes((ctypes.c_char * lib_sz).from_address(lib).raw))
   program = {lib + offset: val for offset, val in program_raw.items()}  # Remap to actual addresses
   lds_size = ((rsrc2 & hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE) >> hsa.AMD_COMPUTE_PGM_RSRC_TWO_GRANULATED_LDS_SIZE_SHIFT) * 512
   total_threads = lx * ly * lz
@@ -1201,12 +1107,6 @@ def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int,
                                  (hsa.AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_Z, gidz)]:
               if rsrc2 & enabled: st._write_sgpr(sgpr_idx, gid); sgpr_idx += 1
 
-            # RDNA4 uses TTMP registers for workgroup IDs: ttmp[9]=gidx, ttmp[10]=gidy, ttmp[11]=gidz
-            if arch == "rdna4":
-              st._write_sgpr(ttmp[9].offset, gidx)
-              st._write_sgpr(ttmp[10].offset, gidy)
-              st._write_sgpr(ttmp[11].offset, gidz)
-
             # v0 = packed workitem IDs, scratch stride in secret SGPR
             for lane in range(n_lanes):
               tid = wave_start + lane
@@ -1223,7 +1123,7 @@ def run_asm(lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int,
               assert fxn is not None, f"[emu] No fxn for {name} at PC={pc}"
               assert 4 not in globals_list or scratch_buf, f"SCRATCH instruction {name} but scratch_size=0"
               if DEBUG >= 6:
-                inst = decode_inst(bytes((ctypes.c_char * 12).from_address(pc).raw), arch)
+                inst = decode_inst(bytes((ctypes.c_char * 12).from_address(pc).raw))
                 print(f"[emu] exec PC={pc:X}: {inst!r}")
               fxn(*[c_bufs[g] for g in globals_list])
             else: raise RuntimeError("exceeded 1M instructions, likely infinite loop")

extra/assembly/amd/pcode.py

@@ -94,19 +94,13 @@ def _trig_reduce(x, phase=0.0):
   return UOp(Ops.SIN, x.dtype, (x - n * _const(x.dtype, 6.283185307179586),))
 
 def _signext(val: UOp) -> UOp:
-  for bits, mask, ext in [(4, 0xF, 0xFFFFFFF0), (8, 0xFF, 0xFFFFFF00), (16, 0xFFFF, 0xFFFF0000)]:
+  for bits, mask, ext in [(8, 0xFF, 0xFFFFFF00), (16, 0xFFFF, 0xFFFF0000)]:
     if (val.op == Ops.AND and len(val.src) == 2 and val.src[1].op == Ops.CONST and val.src[1].arg == mask) or val.dtype.itemsize == bits // 8:
       v32 = val.cast(dtypes.uint32) if val.dtype != dtypes.uint32 else val
       sb = (v32 >> _u32(bits - 1)) & _u32(1)
       return sb.ne(_u32(0)).where(v32 | _u32(ext), v32).cast(dtypes.int)
   return val.cast(dtypes.int64) if val.dtype in (dtypes.int, dtypes.int32) else val
 
-def _signext_4bit(val: UOp) -> UOp:
-  """Sign extend a 4-bit value to 32-bit signed integer."""
-  v32 = val.cast(dtypes.uint32) if val.dtype != dtypes.uint32 else val
-  sb = (v32 >> _u32(3)) & _u32(1)  # sign bit at position 3
-  return sb.ne(_u32(0)).where(v32 | _u32(0xFFFFFFF0), v32).bitcast(dtypes.int)
-
 def _abs(val: UOp) -> UOp:
   if val.dtype not in (dtypes.float32, dtypes.float64, dtypes.half): return val
   _, _, _, _, shift = _float_info(val)
@@ -200,17 +194,6 @@ def _ff1(val: UOp, bits: int) -> UOp:
     result = cond.where(_const(dtypes.int, i), result)
   return result
 
-def _sad_u8(a: UOp, b: UOp, acc: UOp, masked: bool = False) -> UOp:
-  """Sum of absolute differences of 4 unsigned bytes + accumulator. If masked, skips bytes where a == 0."""
-  a, b, acc = a.cast(dtypes.uint32), b.cast(dtypes.uint32), acc.cast(dtypes.uint32)
-  result = acc
-  for i in range(4):
-    a_byte = (a >> _u32(i * 8)) & _u32(0xFF)
-    b_byte = (b >> _u32(i * 8)) & _u32(0xFF)
-    diff = (a_byte > b_byte).where(a_byte - b_byte, b_byte - a_byte)
-    result = result + (a_byte.ne(_u32(0)).where(diff, _u32(0)) if masked else diff)
-  return result
-
 _FUNCS: dict[str, Callable[..., UOp]] = {
   'sqrt': lambda a: UOp(Ops.SQRT, a.dtype, (a,)), 'trunc': lambda a: UOp(Ops.TRUNC, a.dtype, (a,)),
   'log2': lambda a: UOp(Ops.LOG2, a.dtype, (a,)), 'sin': lambda a: _trig_reduce(a),
@@ -244,53 +227,11 @@ _FUNCS: dict[str, Callable[..., UOp]] = {
   'signext_from_bit': _signext_from_bit, 'ldexp': _ldexp, 'frexp_mant': _frexp_mant, 'mantissa': _frexp_mant,
   'frexp_exp': _frexp_exp, 'trig_preop_result': _trig_preop,
   's_ff1_i32_b32': lambda a: _ff1(a, 32), 's_ff1_i32_b64': lambda a: _ff1(a, 64),
-  # Normalization conversions: map [-1,1] or [0,1] to integer range
-  # Use floor(x + 0.5) for round-to-nearest
-  # SNORM: round(value * 32767), range is [-32767, 32767] (hardware behavior)
-  'f16_to_snorm': lambda a: _floor(_f16_extract(a).cast(dtypes.float32) * _const(dtypes.float32, 32767) + _const(dtypes.float32, 0.5)).cast(dtypes.int).cast(dtypes.int16),
-  'f16_to_unorm': lambda a: _floor(_f16_extract(a).cast(dtypes.float32) * _const(dtypes.float32, 65535) + _const(dtypes.float32, 0.5)).cast(dtypes.uint16),
-  'f32_to_snorm': lambda a: _floor(a.bitcast(dtypes.float32) * _const(dtypes.float32, 32767) + _const(dtypes.float32, 0.5)).cast(dtypes.int).cast(dtypes.int16),
-  'f32_to_unorm': lambda a: _floor(a.bitcast(dtypes.float32) * _const(dtypes.float32, 65535) + _const(dtypes.float32, 0.5)).cast(dtypes.uint16),
-  'f32_to_u8': lambda a: _f_to_u(a.bitcast(dtypes.float32), dtypes.uint8),
-  # Integer truncation conversions
-  'i32_to_i16': lambda a: a.cast(dtypes.int).cast(dtypes.int16),
-  'u32_to_u16': lambda a: a.cast(dtypes.uint32).cast(dtypes.uint16),
-  'u16_to_u32': lambda a: (a.cast(dtypes.uint32) & _u32(0xFFFF)),
-  'u8_to_u32': lambda a: (a.cast(dtypes.uint32) & _u32(0xFF)),
-  'u4_to_u32': lambda a: (a.cast(dtypes.uint32) & _u32(0xF)),
-  # Signed extraction with sign extension for dot products
-  'i16_to_i32': lambda a: _signext(a.cast(dtypes.uint32) & _u32(0xFFFF)),
-  'i8_to_i32': lambda a: _signext(a.cast(dtypes.uint32) & _u32(0xFF)),
-  'i4_to_i32': lambda a: _signext_4bit(a.cast(dtypes.uint32) & _u32(0xF)),
-  # Float to int16 conversions
-  'v_cvt_i16_f32': lambda a: UOp(Ops.TRUNC, dtypes.float32, (a.bitcast(dtypes.float32),)).cast(dtypes.int16),
-  'v_cvt_u16_f32': lambda a: _f_to_u(a.bitcast(dtypes.float32), dtypes.uint16),
-  # SAD (Sum of Absolute Differences) - sum |a_i - b_i| for 4 bytes + accumulator
-  'v_sad_u8': lambda a, b, c: _sad_u8(a, b, c),
-  'v_msad_u8': lambda a, b, c: _sad_u8(a, b, c, masked=True),
-  # System NOPs - these are scheduling hints, no effect on emulation
-  'MIN': lambda a, b: (a < b).where(a, b),
-  's_nop': lambda a: _u32(0),
-  # Address calculation for memory operations
-  'CalcDsAddr': lambda a, o, *r: a.cast(dtypes.uint32) + o.cast(dtypes.uint32),
-  'CalcGlobalAddr': lambda v, s, *r: v.cast(dtypes.uint64) + s.cast(dtypes.uint64),
 }
 for is_max, name in [(False, 'min'), (True, 'max')]:
   for dt, sfx in [(dtypes.float32, 'f32'), (dtypes.int, 'i32'), (dtypes.uint32, 'u32'), (dtypes.int16, 'i16'), (dtypes.uint16, 'u16')]:
     _FUNCS[f'v_{name}_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a)
     _FUNCS[f'v_{name}3_{sfx}'] = lambda *a, im=is_max, d=dt: _minmax_reduce(im, d, *a)
-# f16 min/max/min3/max3/med3
-for is_max, name in [(False, 'min'), (True, 'max')]:
-  _FUNCS[f'v_{name}_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a)
-  _FUNCS[f'v_{name}3_num_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}3_num_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a)
-  _FUNCS[f'v_{name}imum_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}imum_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a)
-  _FUNCS[f'v_{name}imum3_f16'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.half, *[_f16_extract(x) for x in a])
-  _FUNCS[f'v_{name}imum3_f32'] = lambda *a, im=is_max: _minmax_reduce(im, dtypes.float32, *a)
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # TOKENIZER/PARSER
@@ -298,7 +239,7 @@ for is_max, name in [(False, 'min'), (True, 'max')]:
 
 DTYPES = {'u32': dtypes.uint32, 'i32': dtypes.int, 'f32': dtypes.float32, 'b32': dtypes.uint32, 'u64': dtypes.uint64, 'i64': dtypes.int64,
           'f64': dtypes.float64, 'b64': dtypes.uint64, 'u16': dtypes.uint16, 'i16': dtypes.short, 'f16': dtypes.half, 'b16': dtypes.uint16,
-          'u8': dtypes.uint8, 'i8': dtypes.int8, 'b8': dtypes.uint8, 'u4': dtypes.uint8, 'i4': dtypes.int8, 'u1': dtypes.uint32}
+          'u8': dtypes.uint8, 'i8': dtypes.int8, 'b8': dtypes.uint8, 'u1': dtypes.uint32}
 _BITS_DT = {8: dtypes.uint8, 16: dtypes.uint16, 32: dtypes.uint32, 64: dtypes.uint64}
 _NUM_SUFFIXES = ('ULL', 'LL', 'UL', 'U', 'L', 'F', 'f')
 def _strip_suffix(num: str) -> tuple[str, str]:
@@ -440,7 +381,7 @@ class Parser:
       self.eat('COMMA')
       lo = self.parse()
       self.eat('RBRACE')
-      return (hi.cast(dt:=_BITS_DT.get((s:=lo.dtype.bitsize) * 2, dtypes.uint64)) << _const(dt, s)) | lo.cast(dt)
+      return (hi.cast(dtypes.uint64) << _u64(32)) | lo.cast(dtypes.uint64)
     if self.at('NUM'):
       num = self.eat('NUM').val
       if self.try_eat('QUOTE'):
@@ -455,7 +396,7 @@ class Parser:
         self.eat('DOT')
         dt_name = self.eat('IDENT').val
         return self._handle_mem_load(addr, DTYPES.get(dt_name, dtypes.uint32))
-      if name == 'VGPR' and self.at('LBRACKET'):
+      if name == 'VGPR':
         self.eat('LBRACKET')
         lane = self.parse()
         self.eat('RBRACKET')
@@ -482,21 +423,7 @@ class Parser:
       if self.try_eat('LBRACE'):
         idx = self.eat('NUM').val
         self.eat('RBRACE')
-        # Handle VGPR{lane}[reg] - 2D array access after loop unrolling
-        if name == 'VGPR' and self.at('LBRACKET'):
-          self.eat('LBRACKET')
-          reg = self.parse()
-          self.eat('RBRACKET')
-          vgpr = self.vars.get('_vgpr')
-          if vgpr is None: return _u32(0)
-          return vgpr.index(_to_u32(reg) * _u32(32) + _u32(int(idx)), ptr=True).load()
-        elem = self.vars.get(f'{name}@{idx}', self.vars.get(f'{name}{idx}'))
-        if elem is None:
-          # Extract bit idx from base variable (like var[idx])
-          base = self.vars.get(name)
-          assert isinstance(base, UOp), f"unknown variable: {name}{idx}"
-          dt = dtypes.uint64 if base.dtype in (dtypes.uint64, dtypes.int64) else dtypes.uint32
-          elem = (base.cast(dt) >> _const(dt, int(idx))) & _const(dt, 1)
+        elem = self.vars.get(f'{name}{idx}', _u32(0))
         if self.try_eat('DOT'):
           dt_name = self.eat('IDENT').val
           return _cast_to(elem, DTYPES.get(dt_name, dtypes.uint32))
@@ -505,17 +432,27 @@ class Parser:
         return elem
       if self.at('LBRACKET') and name not in self.vars:
         self.eat('LBRACKET')
+        if self.at('NUM'):
+          idx_num = int(self.peek().val)
+          if f'{name}{idx_num}' in self.vars:
+            self.eat('NUM')
+            self.eat('RBRACKET')
+            elem = self.vars[f'{name}{idx_num}']
+            if self.try_eat('DOT'): return _cast_to(elem, DTYPES.get(self.eat('IDENT').val, dtypes.uint32))
+            return elem
         first = self.parse()
         return self._handle_bracket_rest(first, _u32(0), name)
       if name in self.vars:
         v = self.vars[name]
-        assert isinstance(v, UOp), f"expected UOp for {name}, got {type(v)}"
-        return v
+        return v if isinstance(v, UOp) else _u32(0) if isinstance(v, dict) else _u32(0)
       raise RuntimeError(f"unknown variable: {name}")
     raise RuntimeError(f"unexpected token in primary: {self.peek()}")
 
   def _handle_dot(self, base, field: str) -> UOp:
-    assert isinstance(base, UOp), f"expected UOp for dot access, got {type(base)}"
+    if isinstance(base, str): return _u32(0)
+    if not isinstance(base, UOp):
+      if isinstance(base, dict): return base.get(field, _u32(0))
+      return _u32(0)
     if field == 'u64' and self.at('LBRACKET') and self.peek(1).type == 'IDENT' and self.peek(1).val == 'laneId':
       self.eat('LBRACKET')
       self.eat_val('laneId', 'IDENT')
@@ -530,7 +467,6 @@ class Parser:
     if dt == base.dtype: return base
     if dt.itemsize == 2 and base.dtype.itemsize == 4:
       return (base & _const(base.dtype, 0xFFFF)).cast(dtypes.uint16) if dt == dtypes.uint16 else (base & _const(base.dtype, 0xFFFF)).cast(dtypes.uint16).bitcast(dt)
-    if field == 'i4': return _signext_4bit(base)
     return _cast_to(base, dt)
 
   def _handle_bracket(self, base, var_name: str | None = None) -> UOp:
@@ -573,18 +509,16 @@ class Parser:
       var_name = self._find_var_name(base)
     if first.op == Ops.CONST:
       idx = int(first.arg)
-      # Check for array element (var@idx)
-      if var_name and f'{var_name}@{idx}' in self.vars:
-        v = self.vars[f'{var_name}@{idx}']
+      if var_name and f'{var_name}{idx}' in self.vars:
+        v = self.vars[f'{var_name}{idx}']
         return _cast_to(v, dt_suffix) if dt_suffix else v
-      # Bit extraction
       dt = dtypes.uint64 if base.dtype in (dtypes.uint64, dtypes.int64) else dtypes.uint32
       base_cast = base.cast(dt) if base.dtype != dt else base
       result = ((base_cast >> _const(dt, idx)) & _const(dt, 1))
       return _cast_to(result, dt_suffix) if dt_suffix else result
     if var_name:
       idx_u32 = _to_u32(first)
-      elems = [(i, self.vars[f'{var_name}@{i}']) for i in range(256) if f'{var_name}@{i}' in self.vars]
+      elems = [(i, self.vars[f'{var_name}{i}']) for i in range(256) if f'{var_name}{i}' in self.vars]
       if elems:
         result = elems[-1][1]
         for ei, ev in reversed(elems[:-1]):
@@ -603,7 +537,7 @@ class Parser:
     self.eat('RBRACE')
     var_name = self._find_var_name(base)
     if var_name:
-      elem = self.vars.get(f'{var_name}@{idx}', _u32(0))  # use @ to avoid collision with temps like A4
+      elem = self.vars.get(f'{var_name}{idx}', _u32(0))
       if self.try_eat('DOT'):
         dt_name = self.eat('IDENT').val
         return _cast_to(elem, DTYPES.get(dt_name, dtypes.uint32))
@@ -665,14 +599,13 @@ class Parser:
     raise RuntimeError(f"unexpected token after {bits}': {self.peek()}")
 
   def _parse_number(self, num: str) -> UOp:
-    if num.startswith('0x') or num.startswith('0X'):
-      is_u64 = num.upper().endswith('ULL') or num.upper().endswith('LL') or num.upper().endswith('UL')
-      return _const(dtypes.uint64 if is_u64 else dtypes.uint32, int(num.rstrip('ULul'), 16))
-    suffix, num_str = _strip_suffix(num)
-    if '.' in num_str or suffix in ('F', 'f'):
-      return _const(dtypes.float32 if suffix in ('F', 'f') else dtypes.float64, float(num_str))
-    val = int(num_str)
-    if 'ULL' in suffix or 'LL' in suffix or 'L' in suffix: return _const(dtypes.uint64, val)
+    if num.startswith('0x') or num.startswith('0X'): return _const(dtypes.uint64, int(num.rstrip('ULul'), 16))
+    suffix, num = _strip_suffix(num)
+    if '.' in num or suffix in ('F', 'f'):
+      return _const(dtypes.float32 if suffix in ('F', 'f') else dtypes.float64, float(num))
+    val = int(num)
+    if 'ULL' in suffix: return _const(dtypes.uint64, val)
+    if 'LL' in suffix or 'L' in suffix: return _const(dtypes.uint64, val)
     if 'U' in suffix: return _const(dtypes.uint32, val)
     return _const(dtypes.int if val < 0 else dtypes.uint32, val)
 
@@ -690,8 +623,7 @@ class Parser:
       if ';' in body or '\n' in body or 'return' in body.lower():
         lines = [l.strip() for l in body.replace(';', '\n').split('\n') if l.strip() and not l.strip().startswith('//')]
         _, _, result = parse_block(lines, 0, lv, self.funcs)
-        assert result is not None, f"lambda {name} must return a value"
-        return result
+        return result if result is not None else _u32(0)
       return parse_expr(body, lv, self.funcs)
     if name in self.funcs:
       return self.funcs[name](*args)
@@ -699,7 +631,7 @@ class Parser:
 
   def _handle_mem_load(self, addr: UOp, dt) -> UOp:
     mem = self.vars.get('_vmem') if '_vmem' in self.vars else self.vars.get('_lds')
-    assert mem is not None, "memory load requires _vmem or _lds"
+    if mem is None: return _const(dt, 0)
     adt = dtypes.uint64 if addr.dtype == dtypes.uint64 else dtypes.uint32
     active = self.vars.get('_active')
     gate = (active,) if active is not None else ()
@@ -761,9 +693,29 @@ def parse_tokens(toks: list[Token], vars: dict[str, VarVal], funcs: dict | None
 
 # Unified block parser for pcode
 def _subst_loop_var(line: str, loop_var: str, val: int) -> str:
-  """Substitute loop variable with its value."""
+  """Substitute loop variable and evaluate bracket expressions.
+  Converts var[loop_var] to var{val} for array element access (like the old regex parser)."""
   toks = tokenize(line)
-  return ' '.join(str(val) if t.type == 'IDENT' and t.val == loop_var else t.val for t in toks if t.type != 'EOF')
+  # First pass: convert var[loop_var] to var{loop_var} to mark for array element assignment
+  result_toks: list[Token] = []
+  j = 0
+  while j < len(toks):
+    t = toks[j]
+    # Check for pattern: IDENT[loop_var] where it's not preceded by a dot (not .type[...])
+    if t.type == 'IDENT' and j+3 < len(toks) and toks[j+1].type == 'LBRACKET' and toks[j+2].type == 'IDENT' and toks[j+2].val == loop_var and toks[j+3].type == 'RBRACKET':
+      # Check that it's not .type[loop_var]
+      if not result_toks or result_toks[-1].type != 'DOT':
+        result_toks.append(t)
+        result_toks.append(Token('LBRACE', '{'))
+        result_toks.append(Token('NUM', str(val)))
+        result_toks.append(Token('RBRACE', '}'))
+        j += 4
+        continue
+    result_toks.append(t)
+    j += 1
+  # Second pass: substitute loop variable in remaining positions
+  subst_parts = [str(val) if t.type == 'IDENT' and t.val == loop_var else t.val for t in result_toks if t.type != 'EOF']
+  return ' '.join(subst_parts)
 
 def _set_bits(old: UOp, val: UOp, width: int, offset: int) -> UOp:
   """Set bits [offset:offset+width) in old to val, masking and shifting appropriately."""
@@ -813,9 +765,8 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
       def parse_bound():
         if p.at('NUM') and p.peek(1).type == 'QUOTE': p.eat('NUM'); p.eat('QUOTE')
         if p.at('NUM'): return int(p.eat('NUM').val.rstrip('UuLl'))
-        expr = p.parse().simplify()
-        assert expr.op == Ops.CONST, f"loop bound must be constant, got {expr}"
-        return int(expr.arg)
+        expr = p.parse()
+        return int(expr.arg) if expr.op == Ops.CONST else 0
       start_val = parse_bound()
       p.eat('COLON')
       end_val = parse_bound()
@@ -836,7 +787,7 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
       if found_var: vars[found_var] = block_assigns[found_var] = _const(dtypes.bool, False)
       for loop_i in range(start_val, end_val + 1):
         subst_lines = [_subst_loop_var(bl, loop_var, loop_i) for bl in body_lines if not (has_break and bl.strip().lower() == 'break')]
-        _, iter_assigns, _ = parse_block(subst_lines, 0, {**vars, **block_assigns}, funcs, assigns)
+        _, iter_assigns, _ = parse_block(subst_lines, 0, vars, funcs, assigns)
         if has_break:
           assert found_var is not None
           found = block_assigns.get(found_var, vars.get(found_var))
@@ -861,9 +812,7 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
 
     # declare
     if first == 'declare':
-      # Initialize scalar declarations (skip arrays and vars already passed as srcs)
-      if '[' not in line and len(toks) >= 2 and toks[1].type == 'IDENT':
-        vars.setdefault(toks[1].val, _u32(0))
+      if '[' not in line and len(toks) >= 2 and toks[1].type == 'IDENT': vars[toks[1].val] = _u32(0)
       i += 1; continue
 
     # lambda definition
@@ -921,7 +870,6 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
       j, lane_toks = _match_bracket(toks, 1)
       if j < len(toks) and toks[j].type == 'LBRACKET':
         j, reg_toks = _match_bracket(toks, j)
-        if j < len(toks) and toks[j].type == 'DOT': j += 2  # skip .type suffix
         if j < len(toks) and toks[j].type == 'EQUALS': j += 1
         ln, rg, val = parse_tokens(lane_toks, vars, funcs), parse_tokens(reg_toks, vars, funcs), parse_tokens(toks[j:], vars, funcs)
         if assigns is not None: assigns.append((f'VGPR[{_tok_str(lane_toks)}][{_tok_str(reg_toks)}]', (_to_u32(rg) * _u32(32) + _to_u32(ln), val)))
@@ -985,32 +933,19 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
             block_assigns[var] = vars[var] = _set_bit(existing, _to_u32(parse_tokens(bit_toks, vars, funcs)), parse_tokens(toks[j+1:], vars, funcs))
             i += 1; continue
 
-    # Array element: var[idx] = value (static index) or var[expr] = value (dynamic)
-    if len(toks) >= 4 and toks[0].type == 'IDENT' and toks[1].type == 'LBRACKET':
-      var = toks[0].val
-      j, idx_toks = _match_bracket(toks, 1)
-      if j < len(toks) and toks[j].type == 'EQUALS':
-        # Static index: var[NUM] = value
-        if len(idx_toks) == 1 and idx_toks[0].type == 'NUM':
-          idx = int(idx_toks[0].val.rstrip('UuLl'))
-          val = parse_tokens(toks[j+1:], vars, funcs)
-          existing = block_assigns.get(var, vars.get(var))
-          if existing is not None and isinstance(existing, UOp):
-            block_assigns[var] = vars[var] = _set_bit(existing, _u32(idx), val)
-          else:
-            block_assigns[f'{var}@{idx}'] = vars[f'{var}@{idx}'] = val
-          i += 1; continue
-        # Dynamic index: var[expr] = value where var has @-elements
-        elems = [(k.split('@')[1], v) for k, v in {**vars, **block_assigns}.items() if k.startswith(f'{var}@') and isinstance(v, UOp)]
-        if elems:
-          idx_expr = parse_tokens(idx_toks, vars, funcs)
-          val = parse_tokens(toks[j+1:], vars, funcs)
-          for elem_idx_str, old_elem in elems:
-            elem_idx = int(elem_idx_str)
-            cond = _to_u32(idx_expr).eq(_u32(elem_idx))
-            new_val = cond.where(val.cast(old_elem.dtype) if val.dtype != old_elem.dtype else val, old_elem)
-            block_assigns[f'{var}@{elem_idx}'] = vars[f'{var}@{elem_idx}'] = new_val
-          i += 1; continue
+    # Array element: var{idx} = value
+    if len(toks) >= 5 and toks[0].type == 'IDENT' and toks[1].type == 'LBRACE' and toks[2].type == 'NUM':
+      var, idx = toks[0].val, int(toks[2].val)
+      j = 4
+      while j < len(toks) and toks[j].type != 'EQUALS': j += 1
+      if j < len(toks):
+        val = parse_tokens(toks[j+1:], vars, funcs)
+        existing = block_assigns.get(var, vars.get(var))
+        if existing is not None and isinstance(existing, UOp):
+          block_assigns[var] = vars[var] = _set_bit(existing, _u32(idx), val)
+        else:
+          block_assigns[f'{var}{idx}'] = vars[f'{var}{idx}'] = val
+        i += 1; continue
 
     # Compound assignment: var += or var -=
     assign_op = next((j for j, t in enumerate(toks) if t.type == 'ASSIGN_OP'), None)
@@ -1057,14 +992,13 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
       def parse_cond(s, kw):
         ll = s.lower()
         return _to_bool(parse_expr(s[ll.find(kw) + len(kw):ll.rfind('then')].strip(), vars, funcs))
-      def is_const(c, v): return c.op == Ops.CONST and c.arg is v
+      def not_static_false(c): return c.op != Ops.CONST or c.arg is not False
       cond = parse_cond(line, 'if')
-      conditions: list[tuple[UOp, UOp | dict[str, VarVal] | None]] = [(cond, None)] if not is_const(cond, False) else []
+      conditions: list[tuple[UOp, UOp | dict[str, VarVal] | None]] = [(cond, None)] if not_static_false(cond) else []
       else_branch: tuple[UOp | None, dict[str, VarVal]] = (None, {})
       vars_snap = dict(vars)
-      static_true = is_const(cond, True)  # track if any condition is statically true
       i += 1
-      i, branch, ret = parse_block(lines, i, vars, funcs, assigns if not is_const(cond, False) else None)
+      i, branch, ret = parse_block(lines, i, vars, funcs, assigns)
       if conditions: conditions[0] = (cond, ret if ret is not None else branch)
       vars.clear(); vars.update(vars_snap)
       while i < len(lines):
@@ -1073,16 +1007,12 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
         lf = ltoks[0].val.lower()
         if lf == 'elsif':
           c = parse_cond(lines[i], 'elsif')
-          take = not static_true and not is_const(c, False)
-          i += 1; i, branch, ret = parse_block(lines, i, vars, funcs, assigns if take else None)
-          if take:
-            conditions.append((c, ret if ret is not None else branch))
-            if is_const(c, True): static_true = True
+          i += 1; i, branch, ret = parse_block(lines, i, vars, funcs, assigns)
+          if not_static_false(c): conditions.append((c, ret if ret is not None else branch))
           vars.clear(); vars.update(vars_snap)
         elif lf == 'else':
-          i += 1
-          i, branch, ret = parse_block(lines, i, vars, funcs, assigns if not static_true else None)
-          if not static_true: else_branch = (ret, branch)
+          i += 1; i, branch, ret = parse_block(lines, i, vars, funcs, assigns)
+          else_branch = (ret, branch)
           vars.clear(); vars.update(vars_snap)
         elif lf == 'endif': i += 1; break
         else: break
@@ -1094,21 +1024,17 @@ def parse_block(lines: list[str], start: int, vars: dict[str, VarVal], funcs: di
             if rv.dtype != result.dtype and rv.dtype.itemsize == result.dtype.itemsize: result = result.cast(rv.dtype)
             result = c.where(rv, result)
         return i, block_assigns, result
-      # If statically true, use that branch directly; otherwise merge with WHERE
-      if static_true:
-        ba = next((b for c, b in conditions if is_const(c, True) and isinstance(b, dict)), {})
-        block_assigns.update(ba); vars.update(ba)
-      else:
-        else_assigns = else_branch[1]
-        all_vars = set().union(*[ba.keys() for _, ba in conditions if isinstance(ba, dict)], else_assigns.keys())
-        for var in all_vars:
-          res: Any = else_assigns.get(var, block_assigns.get(var, vars.get(var, _u32(0))))
-          for cond, ba in reversed(conditions):
-            if isinstance(ba, dict) and var in ba:
-              tv = ba[var]
-              if isinstance(tv, UOp) and isinstance(res, UOp):
-                res = cond.where(tv, res.cast(tv.dtype) if tv.dtype != res.dtype and tv.dtype.itemsize == res.dtype.itemsize else res)
-          block_assigns[var] = vars[var] = res
+      # Main style: merge variable assignments with WHERE
+      else_assigns = else_branch[1]
+      all_vars = set().union(*[ba.keys() for _, ba in conditions if isinstance(ba, dict)], else_assigns.keys())
+      for var in all_vars:
+        res: Any = else_assigns.get(var, block_assigns.get(var, vars.get(var, _u32(0))))
+        for cond, ba in reversed(conditions):
+          if isinstance(ba, dict) and var in ba:
+            tv = ba[var]
+            if isinstance(tv, UOp) and isinstance(res, UOp):
+              res = cond.where(tv, res.cast(tv.dtype) if tv.dtype != res.dtype and tv.dtype.itemsize == res.dtype.itemsize else res)
+        block_assigns[var] = vars[var] = res
       continue
 
     # Regular assignment: var = value

extra/assembly/amd/sqttmap.py

@@ -2,8 +2,11 @@
 from dataclasses import dataclass
 from typing import Iterator
 
+from tinygrad.runtime.support.elf import elf_loader
+
 from extra.assembly.amd.sqtt import decode, print_packets, INST, VALUINST, IMMEDIATE, WAVESTART, WAVEEND, InstOp, PacketType, IMMEDIATE_MASK
 from extra.assembly.amd.dsl import Inst
+from extra.assembly.amd import decode_inst
 from extra.assembly.amd.autogen.rdna3.ins import SOPP, s_endpgm
 from extra.assembly.amd.autogen.rdna3.enum import SOPPOp
 
@@ -13,11 +16,19 @@ class InstructionInfo:
   wave: int
   inst: Inst
 
-def map_insts(data:bytes, lib:bytes, target:int) -> Iterator[tuple[PacketType, InstructionInfo|None]]:
+def map_insts(data:bytes, lib:bytes) -> Iterator[tuple[PacketType, InstructionInfo|None]]:
   """maps SQTT packets to instructions, yields (packet, instruction_info or None)"""
   # map pcs to insts
-  from tinygrad.viz.serve import amd_decode
-  pc_map = amd_decode(lib, target)
+  pc_map:dict[int, Inst] = {}
+  image, sections, _ = elf_loader(lib)
+  text = next((sh for sh in sections if sh.name == ".text"), None)
+  assert text is not None, "no .text section found"
+  text_off, text_size = text.header.sh_addr, text.header.sh_size
+  offset = text_off
+  while offset < text_off + text_size:
+    inst = decode_inst(image[offset:])
+    pc_map[offset-text_off] = inst
+    offset += inst.size()
 
   wave_pc:dict[int, int] = {}
   # only processing packets on one [CU, SIMD] unit
@@ -26,7 +37,7 @@ def map_insts(data:bytes, lib:bytes, target:int) -> Iterator[tuple[PacketType, I
     if not simd_select(p): continue
     if isinstance(p, WAVESTART):
       assert p.wave not in wave_pc, "only one inflight wave per unit"
-      wave_pc[p.wave] = next(iter(pc_map))
+      wave_pc[p.wave] = 0
       continue
     if isinstance(p, WAVEEND):
       pc = wave_pc.pop(p.wave)
@@ -69,22 +80,22 @@ def map_insts(data:bytes, lib:bytes, target:int) -> Iterator[tuple[PacketType, I
 # test to compare every packet with the rocprof decoder
 
 def test_rocprof_inst_traces_match(sqtt, prg, target):
-  from tinygrad.viz.serve import amd_decode
+  from tinygrad.viz.serve import llvm_disasm
   from extra.sqtt.roc import decode as roc_decode, InstExec
-  addr_table = amd_decode(prg.lib, target)
-  disasm = {addr+prg.base:(inst.disasm(), inst.size()) for addr,inst in addr_table.items()}
-  rctx = roc_decode([sqtt], {prg.tag:disasm})
-  rwaves = rctx.inst_execs.get((sqtt.kern, sqtt.exec_tag), [])
+  disasm = {addr+prg.base:inst_disasm for addr, inst_disasm in llvm_disasm(target, prg.lib).items()}
+  rctx = roc_decode([sqtt], {prg.name:disasm})
+  rwaves = rctx.inst_execs[(sqtt.kern, sqtt.exec_tag)]
   rwaves_iter:dict[int, list[Iterator[InstExec]]] = {} # wave unit (0-15) -> list of inst trace iterators for all executions on that unit
   for w in rwaves: rwaves_iter.setdefault(w.wave_id, []).append(w.unpack_insts())
+  rwaves_base = next(iter(disasm)) # base program counter
 
   passed_insts = 0
-  for pkt, info in map_insts(sqtt.blob, prg.lib, target):
+  for pkt, info in map_insts(sqtt.blob, prg.lib):
     if DEBUG >= 2: print_packets([pkt])
     if info is None: continue
     if DEBUG >= 2: print(f"{' '*29}{info.inst.disasm()}")
     rocprof_inst = next(rwaves_iter[info.wave][0])
-    ref_pc = rocprof_inst.pc-prg.base
+    ref_pc = rocprof_inst.pc-rwaves_base
     # always check pc matches
     assert ref_pc == info.pc, f"pc mismatch {ref_pc}:{disasm[rocprof_inst.pc][0]} != {info.pc}:{info.inst.disasm()}"
     # special handling for s_endpgm, it marks the wave completion.
@@ -99,8 +110,7 @@ def test_rocprof_inst_traces_match(sqtt, prg, target):
   for k,v in rwaves_iter.items():
     assert len(v) == 0, f"incomplete wave {k}"
 
-  if len(rwaves):
-    print(f"passed for {passed_insts} instructions across {len(rwaves)} waves scheduled on {len(rwaves_iter)} wave units")
+  print(f"passed for {passed_insts} instructions across {len(rwaves)} waves scheduled on {len(rwaves_iter)} wave units")
 
 if __name__ == "__main__":
   import argparse, pickle, pathlib
@@ -113,7 +123,7 @@ if __name__ == "__main__":
   with open(args.profile, "rb") as f:
     data = pickle.load(f)
   sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
-  kern_events = {e.tag:e for e in data if type(e).__name__ == "ProfileProgramEvent"}
+  kern_events = {e.name:e for e in data if type(e).__name__ == "ProfileProgramEvent"}
   target = next((e for e in data if type(e).__name__ == "ProfileDeviceEvent" and e.device.startswith("AMD"))).props["gfx_target_version"]
   for e in sqtt_events:
     if args.kernel is not None and args.kernel != e.kern: continue

extra/assembly/amd/test/hw/helpers.py

@@ -13,7 +13,7 @@ def _i32(f: float) -> int: return struct.unpack('<I', struct.pack('<f', f))[0]
 def _f32(i: int) -> float: return struct.unpack('<f', struct.pack('<I', i & 0xFFFFFFFF))[0]
 
 # f16 conversion helpers
-def f16(i: int) -> float: return struct.unpack('<e', struct.pack('<H', i & 0xFFFF))[0]
+def _f16(i: int) -> float: return struct.unpack('<e', struct.pack('<H', i & 0xFFFF))[0]
 def f32_to_f16(f: float) -> int:
   f = float(f)
   if math.isnan(f): return 0x7e00
@@ -43,23 +43,6 @@ VCC = VCC_LO  # For VOP3SD sdst field (VCC_LO is exported from dsl)
 USE_HW = os.environ.get("USE_HW", "0") == "1"
 FLOAT_TOLERANCE = 1e-5
 
-def get_gpu_target() -> tuple[int, int, int]:
-  """Get the GPU target as (major, minor, stepping) tuple."""
-  if not USE_HW: return (0, 0, 0)
-  from tinygrad.device import Device
-  return Device["AMD"].target
-
-def skip_unless_gfx(min_major: int, min_minor: int = 0, reason: str = ""):
-  """Skip test if GPU target is below the minimum required version."""
-  import unittest
-  def decorator(test_func):
-    if not USE_HW: return test_func
-    target = get_gpu_target()
-    if target[0] < min_major or (target[0] == min_major and target[1] < min_minor):
-      return unittest.skip(reason or f"requires gfx{min_major}{min_minor}0+")(test_func)
-    return test_func
-  return decorator
-
 # Output buffer layout: vgpr[16][32], sgpr[16], vcc, scc, exec
 N_VGPRS, N_SGPRS, WAVE_SIZE = 16, 16, 32
 VGPR_BYTES = N_VGPRS * WAVE_SIZE * 4  # 16 regs * 32 lanes * 4 bytes = 2048
@@ -229,12 +212,8 @@ amdhsa.kernels:
 
   return parse_output(bytes(out_buf), n_lanes)
 
-def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgprs: int = N_VGPRS, ulp_tolerance: int = 0) -> list[str]:
-  """Compare two WaveStates and return list of differences.
-
-  Args:
-    ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
-  """
+def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgprs: int = N_VGPRS) -> list[str]:
+  """Compare two WaveStates and return list of differences."""
   import math
   diffs = []
   for i in range(n_vgprs):
@@ -245,11 +224,6 @@ def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgp
         emu_f, hw_f = _f32(emu_val), _f32(hw_val)
         if math.isnan(emu_f) and math.isnan(hw_f):
           continue
-        # Check ULP difference for floats (only for same-sign values)
-        if ulp_tolerance > 0 and (emu_val < 0x80000000) == (hw_val < 0x80000000):
-          ulp_diff = abs(int(emu_val) - int(hw_val))
-          if ulp_diff <= ulp_tolerance:
-            continue
         diffs.append(f"v[{i}] lane {lane}: emu=0x{emu_val:08x} ({emu_f:.6g}) hw=0x{hw_val:08x} ({hw_f:.6g})")
   for i in range(N_SGPRS):
     emu_val = emu_st.sgpr[i]
@@ -262,19 +236,16 @@ def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgp
     diffs.append(f"scc: emu={emu_st.scc} hw={hw_st.scc}")
   return diffs
 
-def run_program(instructions: list, n_lanes: int = 1, ulp_tolerance: int = 0) -> WaveState:
+def run_program(instructions: list, n_lanes: int = 1) -> WaveState:
   """Run instructions and return WaveState.
 
   If USE_HW=1, runs on both emulator and hardware, compares results, and raises if they differ.
   Otherwise, runs only on emulator.
-
-  Args:
-    ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
   """
   emu_st = run_program_emu(instructions, n_lanes)
   if USE_HW:
     hw_st = run_program_hw(instructions, n_lanes)
-    diffs = compare_wave_states(emu_st, hw_st, n_lanes, ulp_tolerance=ulp_tolerance)
+    diffs = compare_wave_states(emu_st, hw_st, n_lanes)
     if diffs:
       raise AssertionError(f"Emulator vs Hardware mismatch:\n" + "\n".join(diffs))
     return hw_st

extra/assembly/amd/test/hw/test_ds.py

@@ -719,47 +719,5 @@ class TestAtomicOrdering(unittest.TestCase):
     self.assertEqual(st.vgpr[0][4], 150, "Final value should be 150")
 
 
-class TestDsPermute(unittest.TestCase):
-  """Tests for DS_PERMUTE_B32 and DS_BPERMUTE_B32 instructions."""
-
-  def test_ds_permute_b32_identity(self):
-    """DS_PERMUTE_B32 with identity permutation (lane 0 sends to lane 0)."""
-    # For simplicity, test with single lane
-    instructions = [
-      v_mov_b32_e32(v[0], 0),  # addr = 0 (lane 0)
-      v_mov_b32_e32(v[1], 0xDEADBEEF),  # data
-      ds_permute_b32(v[2], v[0], v[1]),
-      s_waitcnt(lgkmcnt=0),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    # Lane 0 sends to lane 0, so lane 0 gets 0xDEADBEEF
-    self.assertEqual(st.vgpr[0][2], 0xDEADBEEF)
-
-  def test_ds_bpermute_b32_identity(self):
-    """DS_BPERMUTE_B32 with identity permutation (each lane reads from itself)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),  # addr = 0 (read from lane 0)
-      v_mov_b32_e32(v[1], 0xCAFEBABE),  # data in lane 0
-      ds_bpermute_b32(v[2], v[0], v[1]),
-      s_waitcnt(lgkmcnt=0),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    # Lane 0 reads from lane 0's v[1]
-    self.assertEqual(st.vgpr[0][2], 0xCAFEBABE)
-
-  def test_ds_permute_b32_broadcast(self):
-    """DS_PERMUTE_B32 broadcast - all lanes send to lane 0."""
-    # With 4 lanes, all sending to lane 0, highest lane wins
-    instructions = [
-      v_mov_b32_e32(v[0], 0),  # All lanes send to addr 0 (lane 0)
-      v_mov_b32_e32(v[1], 0x11111111),  # All lanes send same data
-      ds_permute_b32(v[2], v[0], v[1]),
-      s_waitcnt(lgkmcnt=0),
-    ]
-    st = run_program(instructions, n_lanes=4)
-    # Lane 0 receives data (highest numbered active lane wins)
-    self.assertEqual(st.vgpr[0][2], 0x11111111)
-
-
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_sop.py

@@ -62,7 +62,6 @@ class TestBasicScalar(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     self.assertEqual(st.sgpr[1], 0x80000000)
 
-  @skip_unless_gfx(11, 5, "SALU FP ops require gfx1150+")
   def test_s_fmamk_f32(self):
     """S_FMAMK_F32: D = S0 * literal + S1."""
     # 2.0 * 3.0 + 1.0 = 7.0
@@ -74,7 +73,6 @@ class TestBasicScalar(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     self.assertEqual(st.sgpr[2], f2i(7.0))
 
-  @skip_unless_gfx(11, 5, "SALU FP ops require gfx1150+")
   def test_s_fmamk_f32_negative(self):
     """S_FMAMK_F32 with negative values."""
     # -2.0 * 4.0 + 10.0 = 2.0
@@ -87,50 +85,6 @@ class TestBasicScalar(unittest.TestCase):
     self.assertEqual(st.sgpr[2], f2i(2.0))
 
 
-class TestPack(unittest.TestCase):
-  """Tests for S_PACK instructions."""
-
-  def test_s_pack_ll_b32_b16(self):
-    """S_PACK_LL_B32_B16 packs low 16 bits of two sources into one 32-bit result."""
-    instructions = [
-      s_mov_b32(s[0], 0xDEADAAAA),
-      s_mov_b32(s[1], 0xDEADBBBB),
-      s_pack_ll_b32_b16(s[2], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.sgpr[2], 0xBBBBAAAA)
-
-  def test_s_pack_lh_b32_b16(self):
-    """S_PACK_LH_B32_B16: D0 = { S1[31:16], S0[15:0] }."""
-    instructions = [
-      s_mov_b32(s[0], 0xDEADAAAA),
-      s_mov_b32(s[1], 0xDEADBBBB),
-      s_pack_lh_b32_b16(s[2], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.sgpr[2], 0xDEADAAAA)
-
-  def test_s_pack_hh_b32_b16(self):
-    """S_PACK_HH_B32_B16: D0 = { S1[31:16], S0[31:16] }."""
-    instructions = [
-      s_mov_b32(s[0], 0xDEADAAAA),
-      s_mov_b32(s[1], 0xDEADBBBB),
-      s_pack_hh_b32_b16(s[2], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.sgpr[2], 0xDEADDEAD)
-
-  def test_s_pack_hl_b32_b16(self):
-    """S_PACK_HL_B32_B16: D0 = { S1[15:0], S0[31:16] }."""
-    instructions = [
-      s_mov_b32(s[0], 0xDEADAAAA),
-      s_mov_b32(s[1], 0xDEADBBBB),
-      s_pack_hl_b32_b16(s[2], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.sgpr[2], 0xBBBBDEAD)
-
-
 class TestQuadmaskWqm(unittest.TestCase):
   """Tests for S_QUADMASK_B32 and S_WQM_B32."""
 

extra/assembly/amd/test/hw/test_vop1.py

@@ -255,6 +255,7 @@ class TestF16Conversions(unittest.TestCase):
 
   def test_v_cvt_f16_f32_small(self):
     """V_CVT_F16_F32 converts small f32 value."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     instructions = [
       v_mov_b32_e32(v[0], 0.5),
       v_cvt_f16_f32_e32(v[1], v[0]),
@@ -292,6 +293,7 @@ class TestF16Conversions(unittest.TestCase):
 
   def test_v_cvt_f16_f32_reads_full_32bit_source(self):
     """V_CVT_F16_F32 must read full 32-bit f32 source."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x3fc00000),  # f32 1.5
       v_mov_b32_e32(v[0], s[0]),
@@ -300,7 +302,7 @@ class TestF16Conversions(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
     lo_bits = result & 0xffff
-    self.assertEqual(lo_bits, 0x3e00, f"Expected f16(1.5)=0x3e00, got 0x{lo_bits:04x} ({f16(lo_bits)})")
+    self.assertEqual(lo_bits, 0x3e00, f"Expected f16(1.5)=0x3e00, got 0x{lo_bits:04x} ({_f16(lo_bits)})")
 
   def test_v_cvt_i16_f16_zero(self):
     """V_CVT_I16_F16 converts f16 zero to i16 zero."""
@@ -694,6 +696,7 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_abs_negative(self):
     """V_CVT_F32_F16 with |abs| on negative value."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
     instructions = [
       s_mov_b32(s[0], f16_neg1),
@@ -706,6 +709,7 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_abs_positive(self):
     """V_CVT_F32_F16 with |abs| on positive value (should stay positive)."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_2 = f32_to_f16(2.0)  # 0x4000
     instructions = [
       s_mov_b32(s[0], f16_2),
@@ -718,6 +722,7 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_neg_positive(self):
     """V_CVT_F32_F16 with neg on positive value."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_2 = f32_to_f16(2.0)  # 0x4000
     instructions = [
       s_mov_b32(s[0], f16_2),
@@ -730,6 +735,7 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_neg_negative(self):
     """V_CVT_F32_F16 with neg on negative value (double negative)."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_neg2 = f32_to_f16(-2.0)  # 0xc000
     instructions = [
       s_mov_b32(s[0], f16_neg2),
@@ -742,6 +748,7 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f16_f32_then_pack_for_wmma(self):
     """CVT F32->F16 followed by pack (common WMMA pattern)."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     f32_val = 3.5
     instructions = [
       s_mov_b32(s[0], f2i(f32_val)),
@@ -750,8 +757,8 @@ class TestCvtF16Modifiers(unittest.TestCase):
       v_pack_b32_f16(v[2], v[1], v[1]),  # Pack same value
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = f16(st.vgpr[0][2] & 0xffff)
-    hi = f16((st.vgpr[0][2] >> 16) & 0xffff)
+    lo = _f16(st.vgpr[0][2] & 0xffff)
+    hi = _f16((st.vgpr[0][2] >> 16) & 0xffff)
     self.assertAlmostEqual(lo, f32_val, places=1)
     self.assertAlmostEqual(hi, f32_val, places=1)
 
@@ -797,6 +804,7 @@ class TestConversionRounding(unittest.TestCase):
 
   def test_f16_to_f32_precision(self):
     """F16 to F32 conversion precision."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_val = f32_to_f16(1.5)
     instructions = [
       s_mov_b32(s[0], f16_val),
@@ -808,6 +816,7 @@ class TestConversionRounding(unittest.TestCase):
 
   def test_f16_denormal_to_f32(self):
     """F16 denormal converts to small positive f32."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     f16_denorm = 0x0001  # Smallest positive f16 denormal
     instructions = [
       v_mov_b32_e32(v[0], f16_denorm),
@@ -1503,82 +1512,5 @@ class TestReciprocalF16(unittest.TestCase):
     self.assertAlmostEqual(result, 0.25, places=2, msg="1/4.0 should be 0.25")
 
 
-class TestCvtNormF16(unittest.TestCase):
-  """Tests for V_CVT_NORM_I16_F16 and V_CVT_NORM_U16_F16."""
-
-  def test_cvt_norm_i16_f16_positive(self):
-    """V_CVT_NORM_I16_F16: f16 1.0 -> i16 max (32767)."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(1.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_cvt_norm_i16_f16_e32(v[1], v[0]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1] & 0xffff
-    self.assertEqual(result, 32767)
-
-  def test_cvt_norm_i16_f16_negative(self):
-    """V_CVT_NORM_I16_F16: f16 -1.0 -> i16 -32767 (0x8001)."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(-1.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_cvt_norm_i16_f16_e32(v[1], v[0]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1] & 0xffff
-    self.assertEqual(result, 0x8001)  # -32767, hardware uses symmetric range
-
-  def test_cvt_norm_i16_f16_zero(self):
-    """V_CVT_NORM_I16_F16: f16 0.0 -> i16 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),
-      v_cvt_norm_i16_f16_e32(v[1], v[0]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1] & 0xffff
-    self.assertEqual(result, 0)
-
-  def test_cvt_norm_u16_f16_one(self):
-    """V_CVT_NORM_U16_F16: f16 1.0 -> u16 max (65535)."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(1.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_cvt_norm_u16_f16_e32(v[1], v[0]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1] & 0xffff
-    self.assertEqual(result, 65535)
-
-  def test_cvt_norm_u16_f16_half(self):
-    """V_CVT_NORM_U16_F16: f16 0.5 -> u16 ~32768."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(0.5)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_cvt_norm_u16_f16_e32(v[1], v[0]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1] & 0xffff
-    self.assertAlmostEqual(result, 32768, delta=1)
-
-
-class TestPermlane64(unittest.TestCase):
-  """Tests for V_PERMLANE64_B32 instruction (wave64 cross-half swap)."""
-
-  def test_v_permlane64_b32_is_nop_in_wave32(self):
-    """V_PERMLANE64_B32 is a NOP in wave32 mode.
-
-    Per AMD pcode: "if WAVE32 then s_nop(...) else ... endif"
-    The emulator runs in wave32 mode, so this instruction should not modify registers.
-    """
-    instructions = [
-      v_mov_b32_e32(v[0], 0xCAFEBABE),  # source
-      v_mov_b32_e32(v[1], 0x12345678),  # dest (should be preserved)
-      v_permlane64_b32_e32(v[1], v[0]),  # NOP in wave32
-    ]
-    st = run_program(instructions, n_lanes=1)
-    # Dest register should be unchanged (NOP behavior in wave32)
-    self.assertEqual(st.vgpr[0][1], 0x12345678)
-
-
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vop3.py

@@ -857,6 +857,7 @@ class TestF16Modifiers(unittest.TestCase):
 
   def test_v_fma_f16_inline_const_1_0(self):
     """V_FMA_F16: a*b + 1.0 should use f16 inline constant."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16, _f16
     f16_a = f32_to_f16(0.325928)  # ~0x3537
     f16_b = f32_to_f16(-0.486572)  # ~0xb7c9
     instructions = [
@@ -867,12 +868,13 @@ class TestF16Modifiers(unittest.TestCase):
       v_fma_f16(v[4], v[4], v[6], 1.0),  # 1.0 is inline constant
     ]
     st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][4] & 0xffff)
+    result = _f16(st.vgpr[0][4] & 0xffff)
     expected = 0.325928 * (-0.486572) + 1.0
     self.assertAlmostEqual(result, expected, delta=0.01)
 
   def test_v_fma_f16_inline_const_0_5(self):
     """V_FMA_F16: a*b + 0.5 should use f16 inline constant."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16, _f16
     f16_a = f32_to_f16(2.0)
     f16_b = f32_to_f16(3.0)
     instructions = [
@@ -883,12 +885,13 @@ class TestF16Modifiers(unittest.TestCase):
       v_fma_f16(v[2], v[0], v[1], 0.5),  # 0.5 is inline constant
     ]
     st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][2] & 0xffff)
+    result = _f16(st.vgpr[0][2] & 0xffff)
     expected = 2.0 * 3.0 + 0.5
     self.assertAlmostEqual(result, expected, delta=0.01)
 
   def test_v_fma_f16_inline_const_neg_1_0(self):
     """V_FMA_F16: a*b + (-1.0) should use f16 inline constant."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16, _f16
     f16_a = f32_to_f16(2.0)
     f16_b = f32_to_f16(3.0)
     instructions = [
@@ -899,12 +902,13 @@ class TestF16Modifiers(unittest.TestCase):
       v_fma_f16(v[2], v[0], v[1], -1.0),  # -1.0 is inline constant
     ]
     st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][2] & 0xffff)
+    result = _f16(st.vgpr[0][2] & 0xffff)
     expected = 2.0 * 3.0 + (-1.0)
     self.assertAlmostEqual(result, expected, delta=0.01)
 
   def test_v_add_f16_abs_both(self):
     """V_ADD_F16 with abs on both operands."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16, _f16
     f16_neg2 = f32_to_f16(-2.0)
     f16_neg3 = f32_to_f16(-3.0)
     instructions = [
@@ -915,11 +919,12 @@ class TestF16Modifiers(unittest.TestCase):
       v_add_f16_e64(v[2], abs(v[0]), abs(v[1])),  # |-2| + |-3| = 5
     ]
     st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][2] & 0xffff)
+    result = _f16(st.vgpr[0][2] & 0xffff)
     self.assertAlmostEqual(result, 5.0, delta=0.01)
 
   def test_v_mul_f16_neg_abs(self):
     """V_MUL_F16 with neg on one operand and abs on another."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16, _f16
     f16_2 = f32_to_f16(2.0)
     f16_neg3 = f32_to_f16(-3.0)
     instructions = [
@@ -930,7 +935,7 @@ class TestF16Modifiers(unittest.TestCase):
       v_mul_f16_e64(v[2], -v[0], abs(v[1])),  # -(2) * |-3| = -6
     ]
     st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][2] & 0xffff)
+    result = _f16(st.vgpr[0][2] & 0xffff)
     self.assertAlmostEqual(result, -6.0, delta=0.01)
 
   def test_v_fmac_f16_hi_dest(self):
@@ -938,6 +943,7 @@ class TestF16Modifiers(unittest.TestCase):
 
     This tests the case from AMD_LLVM sin(0) where V_FMAC_F16 writes to v0.h.
     """
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x38003c00),  # v0 = {hi=0.5, lo=1.0}
       v_mov_b32_e32(v[0], s[0]),
@@ -948,8 +954,8 @@ class TestF16Modifiers(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     v0 = st.vgpr[0][0]
-    result_hi = f16((v0 >> 16) & 0xffff)
-    result_lo = f16(v0 & 0xffff)
+    result_hi = _f16((v0 >> 16) & 0xffff)
+    result_lo = _f16(v0 & 0xffff)
     self.assertAlmostEqual(result_hi, 0.5, delta=0.01, msg=f"Expected hi=0.5, got {result_hi}")
     self.assertAlmostEqual(result_lo, 1.0, delta=0.01, msg=f"Expected lo=1.0, got {result_lo}")
 
@@ -2949,694 +2955,5 @@ class TestVOP3Clamp(unittest.TestCase):
     self.assertAlmostEqual(i2f(st.vgpr[3][1]), 1.0, places=5, msg="lane 3: 2.5 should clamp to 1.0")
 
 
-class TestVOP3ClampUint32(unittest.TestCase):
-  """Tests for VOP3 clamp modifier on unsigned 32-bit integer operations."""
-
-  def test_v_sub_nc_u32_e64_clamp_underflow(self):
-    """V_SUB_NC_U32_E64 with clamp: 0 - 1 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),
-      v_mov_b32_e32(v[1], 1),
-      v_sub_nc_u32_e64(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0, f"expected 0, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_sub_nc_u32_e64_clamp_no_underflow(self):
-    """V_SUB_NC_U32_E64 with clamp: 100 - 50 = 50 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 50),
-      v_sub_nc_u32_e64(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 50, f"expected 50, got {st.vgpr[0][2]}")
-
-  def test_v_add_nc_u32_e64_clamp_overflow(self):
-    """V_ADD_NC_U32_E64 with clamp: 0xFFFFFFFF + 1 should saturate to 0xFFFFFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFFFFFF),
-      v_mov_b32_e32(v[1], 1),
-      v_add_nc_u32_e64(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0xFFFFFFFF, f"expected 0xFFFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_nc_u32_e64_clamp_no_overflow(self):
-    """V_ADD_NC_U32_E64 with clamp: 100 + 50 = 150 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 50),
-      v_add_nc_u32_e64(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 150, f"expected 150, got {st.vgpr[0][2]}")
-
-
-class TestVOP3ClampUint16(unittest.TestCase):
-  """Tests for VOP3 clamp modifier on unsigned 16-bit integer operations."""
-
-  def test_v_sub_nc_u16_clamp_underflow(self):
-    """V_SUB_NC_U16 with clamp: 0 - 1 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),
-      v_mov_b32_e32(v[1], 1),
-      v_sub_nc_u16(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2] & 0xFFFF, 0, f"expected 0, got 0x{st.vgpr[0][2] & 0xFFFF:04x}")
-
-  def test_v_sub_nc_u16_clamp_no_underflow(self):
-    """V_SUB_NC_U16 with clamp: 100 - 50 = 50 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 50),
-      v_sub_nc_u16(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2] & 0xFFFF, 50, f"expected 50, got {st.vgpr[0][2] & 0xFFFF}")
-
-  def test_v_add_nc_u16_clamp_overflow(self):
-    """V_ADD_NC_U16 with clamp: 0xFFFF + 1 should saturate to 0xFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFF),
-      v_mov_b32_e32(v[1], 1),
-      v_add_nc_u16(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2] & 0xFFFF, 0xFFFF, f"expected 0xFFFF, got 0x{st.vgpr[0][2] & 0xFFFF:04x}")
-
-  def test_v_add_nc_u16_clamp_no_overflow(self):
-    """V_ADD_NC_U16 with clamp: 100 + 50 = 150 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 50),
-      v_add_nc_u16(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2] & 0xFFFF, 150, f"expected 150, got {st.vgpr[0][2] & 0xFFFF}")
-
-
-class TestVOP3ClampInt32(unittest.TestCase):
-  """Tests for VOP3 clamp modifier on signed 32-bit integer operations."""
-
-  def test_v_add_nc_i32_clamp_overflow(self):
-    """V_ADD_NC_I32 with clamp: INT_MAX + 1 should saturate to INT_MAX."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x7FFFFFFF),  # S0 = INT_MAX
-      v_mov_b32_e32(v[1], 1),            # S1 = 1
-      v_add_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0x7FFFFFFF, f"expected 0x7FFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_nc_i32_clamp_underflow(self):
-    """V_ADD_NC_I32 with clamp: INT_MIN + (-1) should saturate to INT_MIN."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x80000000),  # S0 = INT_MIN
-      v_mov_b32_e32(v[1], 0xFFFFFFFF),  # S1 = -1
-      v_add_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0x80000000, f"expected 0x80000000, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_sub_nc_i32_clamp_underflow(self):
-    """V_SUB_NC_I32 with clamp: INT_MIN - 1 should saturate to INT_MIN."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x80000000),  # S0 = INT_MIN
-      v_mov_b32_e32(v[1], 1),            # S1 = 1
-      v_sub_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0x80000000, f"expected 0x80000000, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_sub_nc_i32_clamp_overflow(self):
-    """V_SUB_NC_I32 with clamp: INT_MAX - (-1) should saturate to INT_MAX."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x7FFFFFFF),  # S0 = INT_MAX
-      v_mov_b32_e32(v[1], 0xFFFFFFFF),  # S1 = -1
-      v_sub_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0x7FFFFFFF, f"expected 0x7FFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_nc_i32_no_saturation_positive(self):
-    """V_ADD_NC_I32 with clamp: 100 + 200 = 300 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 200),
-      v_add_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 300, f"expected 300, got {st.vgpr[0][2]}")
-
-  def test_v_add_nc_i32_no_saturation_negative(self):
-    """V_ADD_NC_I32 with clamp: -100 + -200 = -300 (no saturation needed)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFFFF9C),  # -100
-      v_mov_b32_e32(v[1], 0xFFFFFF38),  # -200
-      v_add_nc_i32(v[2], v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    expected = 0xFFFFFED4  # -300
-    self.assertEqual(st.vgpr[0][2], expected, f"expected 0x{expected:08x}, got 0x{st.vgpr[0][2]:08x}")
-
-
-class TestVOP3ClampCarry(unittest.TestCase):
-  """Tests for VOP3 clamp modifier on carry operations (VOP3SD)."""
-
-  def test_v_add_co_u32_clamp_overflow(self):
-    """V_ADD_CO_U32 with clamp: 0xFFFFFFFF + 1 should saturate to 0xFFFFFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFFFFFF),
-      v_mov_b32_e32(v[1], 1),
-      v_add_co_u32(v[2], VCC, v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0xFFFFFFFF, f"expected 0xFFFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_co_u32_clamp_no_overflow(self):
-    """V_ADD_CO_U32 with clamp: 100 + 200 = 300 (no saturation)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 200),
-      v_add_co_u32(v[2], VCC, v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 300, f"expected 300, got {st.vgpr[0][2]}")
-
-  def test_v_sub_co_u32_clamp_underflow(self):
-    """V_SUB_CO_U32 with clamp: 0 - 1 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),
-      v_mov_b32_e32(v[1], 1),
-      v_sub_co_u32(v[2], VCC, v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0, f"expected 0, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_sub_co_u32_clamp_no_underflow(self):
-    """V_SUB_CO_U32 with clamp: 300 - 100 = 200 (no saturation)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 300),
-      v_mov_b32_e32(v[1], 100),
-      v_sub_co_u32(v[2], VCC, v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 200, f"expected 200, got {st.vgpr[0][2]}")
-
-  def test_v_subrev_co_u32_clamp_underflow(self):
-    """V_SUBREV_CO_U32 with clamp: 1 - 0 reversed = 0 - 1 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 1),  # This becomes the subtrahend
-      v_mov_b32_e32(v[1], 0),  # This becomes the minuend (0 - 1)
-      v_subrev_co_u32(v[2], VCC, v[0], v[1], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0, f"expected 0, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_co_ci_u32_clamp_overflow(self):
-    """V_ADD_CO_CI_U32 with clamp: 0xFFFFFFFF + 1 + 0 should saturate to 0xFFFFFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFFFFFF),
-      v_mov_b32_e32(v[1], 1),
-      s_mov_b64(VCC, 0),  # No carry in
-      v_add_co_ci_u32(v[2], VCC, v[0], v[1], VCC, clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0xFFFFFFFF, f"expected 0xFFFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_add_co_ci_u32_clamp_overflow_with_carry(self):
-    """V_ADD_CO_CI_U32 with clamp: 0xFFFFFFFE + 1 + 1 should saturate to 0xFFFFFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFFFFFE),
-      v_mov_b32_e32(v[1], 1),
-      s_mov_b64(VCC, 1),  # Carry in = 1
-      v_add_co_ci_u32(v[2], VCC, v[0], v[1], VCC, clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0xFFFFFFFF, f"expected 0xFFFFFFFF, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_sub_co_ci_u32_clamp_underflow(self):
-    """V_SUB_CO_CI_U32 with clamp: 0 - 1 - 0 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0),
-      v_mov_b32_e32(v[1], 1),
-      s_mov_b64(VCC, 0),  # No borrow in
-      v_sub_co_ci_u32(v[2], VCC, v[0], v[1], VCC, clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0, f"expected 0, got 0x{st.vgpr[0][2]:08x}")
-
-  def test_v_subrev_co_ci_u32_clamp_underflow(self):
-    """V_SUBREV_CO_CI_U32 with clamp: reversed 1 - 0 - 0 = 0 - 1 should saturate to 0."""
-    instructions = [
-      v_mov_b32_e32(v[0], 1),
-      v_mov_b32_e32(v[1], 0),
-      s_mov_b64(VCC, 0),
-      v_subrev_co_ci_u32(v[2], VCC, v[0], v[1], VCC, clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][2], 0, f"expected 0, got 0x{st.vgpr[0][2]:08x}")
-
-
-class TestVOP3ClampMAD(unittest.TestCase):
-  """Tests for VOP3 clamp modifier on MAD (multiply-add) operations."""
-
-  def test_v_mad_u16_clamp_overflow(self):
-    """V_MAD_U16 with clamp: 0xFFFF * 2 + 0 should saturate to 0xFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFF),
-      v_mov_b32_e32(v[1], 2),
-      v_mov_b32_e32(v[2], 0),
-      v_mad_u16(v[3], v[0], v[1], v[2], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3] & 0xFFFF, 0xFFFF, f"expected 0xFFFF, got 0x{st.vgpr[0][3] & 0xFFFF:04x}")
-
-  def test_v_mad_u16_clamp_overflow_with_add(self):
-    """V_MAD_U16 with clamp: 0x8000 * 2 + 0x1000 should saturate to 0xFFFF."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x8000),  # 32768
-      v_mov_b32_e32(v[1], 2),        # * 2 = 65536
-      v_mov_b32_e32(v[2], 0x1000),   # + 4096 = 69632 > 0xFFFF
-      v_mad_u16(v[3], v[0], v[1], v[2], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3] & 0xFFFF, 0xFFFF, f"expected 0xFFFF, got 0x{st.vgpr[0][3] & 0xFFFF:04x}")
-
-  def test_v_mad_u16_no_overflow(self):
-    """V_MAD_U16 with clamp: 100 * 100 + 50 = 10050 (no saturation)."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100),
-      v_mov_b32_e32(v[1], 100),
-      v_mov_b32_e32(v[2], 50),
-      v_mad_u16(v[3], v[0], v[1], v[2], clmp=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3] & 0xFFFF, 10050, f"expected 10050, got {st.vgpr[0][3] & 0xFFFF}")
-
-  def test_v_mad_u16_no_clamp(self):
-    """V_MAD_U16 without clamp: 0xFFFF * 2 + 0 should wrap to 0xFFFE."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0xFFFF),
-      v_mov_b32_e32(v[1], 2),
-      v_mov_b32_e32(v[2], 0),
-      v_mad_u16(v[3], v[0], v[1], v[2], clmp=0),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    # 0xFFFF * 2 = 0x1FFFE, low 16 bits = 0xFFFE
-    self.assertEqual(st.vgpr[0][3] & 0xFFFF, 0xFFFE, f"expected 0xFFFE, got 0x{st.vgpr[0][3] & 0xFFFF:04x}")
-
-
-class TestCvtPkF16(unittest.TestCase):
-  """Tests for V_CVT_PK_RTZ_F16_F32 - pack two f32 to f16 with round toward zero."""
-
-  def test_cvt_pk_rtz_f16_f32_basic(self):
-    """V_CVT_PK_RTZ_F16_F32: basic pack of two f32 values."""
-    instructions = [
-      v_mov_b32_e32(v[0], 1.0),
-      v_mov_b32_e32(v[1], 2.0),
-      v_cvt_pk_rtz_f16_f32_e64(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo_f16 = f16(result & 0xffff)
-    hi_f16 = f16((result >> 16) & 0xffff)
-    self.assertAlmostEqual(lo_f16, 1.0, delta=0.01)
-    self.assertAlmostEqual(hi_f16, 2.0, delta=0.01)
-
-
-class TestCvtPkNorm(unittest.TestCase):
-  """Tests for V_CVT_PK_NORM_I16_F32 and V_CVT_PK_NORM_U16_F32."""
-
-  def test_cvt_pk_norm_i16_f32_basic(self):
-    """V_CVT_PK_NORM_I16_F32: pack two f32 to normalized i16."""
-    instructions = [
-      v_mov_b32_e32(v[0], 1.0),
-      v_mov_b32_e32(v[1], -1.0),
-      v_cvt_pk_norm_i16_f32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    self.assertEqual(lo, 32767)
-    self.assertEqual(hi, 0x8001)  # -32767, hardware uses symmetric range
-
-  def test_cvt_pk_norm_u16_f32_basic(self):
-    """V_CVT_PK_NORM_U16_F32: pack two f32 to normalized u16."""
-    instructions = [
-      v_mov_b32_e32(v[0], 1.0),
-      v_mov_b32_e32(v[1], 0.5),
-      v_cvt_pk_norm_u16_f32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    self.assertEqual(lo, 65535)
-    self.assertAlmostEqual(hi, 32768, delta=1)
-
-
-class TestCvtPkInt(unittest.TestCase):
-  """Tests for V_CVT_PK_I16_I32, V_CVT_PK_U16_U32, V_CVT_PK_I16_F32, V_CVT_PK_U16_F32."""
-
-  def test_cvt_pk_i16_i32_basic(self):
-    """V_CVT_PK_I16_I32: pack two i32 to i16."""
-    instructions = [
-      s_mov_b32(s[0], 100),
-      s_mov_b32(s[1], -100 & 0xffffffff),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_cvt_pk_i16_i32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    lo_signed = lo if lo < 32768 else lo - 65536
-    hi_signed = hi if hi < 32768 else hi - 65536
-    self.assertEqual(lo_signed, 100)
-    self.assertEqual(hi_signed, -100)
-
-  def test_cvt_pk_u16_u32_basic(self):
-    """V_CVT_PK_U16_U32: pack two u32 to u16."""
-    instructions = [
-      s_mov_b32(s[0], 1000),
-      s_mov_b32(s[1], 2000),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_cvt_pk_u16_u32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    self.assertEqual(lo, 1000)
-    self.assertEqual(hi, 2000)
-
-  def test_cvt_pk_i16_f32_basic(self):
-    """V_CVT_PK_I16_F32: convert two f32 to packed i16."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100.5),
-      v_mov_b32_e32(v[1], -50.7),
-      v_cvt_pk_i16_f32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    lo_signed = lo if lo < 32768 else lo - 65536
-    hi_signed = hi if hi < 32768 else hi - 65536
-    self.assertEqual(lo_signed, 100)
-    self.assertEqual(hi_signed, -50)
-
-  def test_cvt_pk_u16_f32_basic(self):
-    """V_CVT_PK_U16_F32: convert two f32 to packed u16."""
-    instructions = [
-      v_mov_b32_e32(v[0], 100.9),
-      v_mov_b32_e32(v[1], 200.1),
-      v_cvt_pk_u16_f32(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    self.assertEqual(lo, 100)
-    self.assertEqual(hi, 200)
-
-  def test_cvt_pk_u8_f32_basic(self):
-    """V_CVT_PK_U8_F32: convert f32 to u8 and pack at byte position."""
-    instructions = [
-      v_mov_b32_e32(v[0], 128.5),
-      v_mov_b32_e32(v[1], 0),
-      v_mov_b32_e32(v[2], 0),
-      v_cvt_pk_u8_f32(v[2], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    byte0 = result & 0xff
-    self.assertEqual(byte0, 128)
-
-
-class TestDotProduct(unittest.TestCase):
-  """Tests for dot product instructions V_DOT4_U32_U8, V_DOT8_U32_U4."""
-
-  def test_v_dot4_u32_u8_basic(self):
-    """V_DOT4_U32_U8: 4-element dot product of u8 vectors."""
-    src0 = 0x04030201  # {4, 3, 2, 1}
-    src1 = 0x01010101  # {1, 1, 1, 1}
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot4_u32_u8(v[2], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    self.assertEqual(result, 10)
-
-  def test_v_dot4_u32_u8_with_accumulator(self):
-    """V_DOT4_U32_U8 with non-zero accumulator."""
-    src0 = 0x02020202  # {2, 2, 2, 2}
-    src1 = 0x03030303  # {3, 3, 3, 3}
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 100),
-      v_dot4_u32_u8(v[2], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    self.assertEqual(result, 124)
-
-  def test_v_dot8_u32_u4_basic(self):
-    """V_DOT8_U32_U4: 8-element dot product of u4 vectors."""
-    # src0 = 8 nibbles: {1,2,3,4,5,6,7,8} packed as 0x87654321
-    # src1 = 8 nibbles: {1,1,1,1,1,1,1,1} packed as 0x11111111
-    # result = 1+2+3+4+5+6+7+8 = 36
-    src0 = 0x87654321
-    src1 = 0x11111111
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot8_u32_u4(v[2], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    self.assertEqual(result, 36)
-
-
-class TestMinMaxF16Vop3(unittest.TestCase):
-  """Tests for V_MIN3_F16, V_MAX3_F16, V_MED3_F16, V_MINMAX_F16, V_MAXMIN_F16."""
-
-  def test_v_min3_f16_basic(self):
-    """V_MIN3_F16: minimum of three f16 values."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(3.0)),
-      s_mov_b32(s[1], f32_to_f16(1.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_min3_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 1.0, delta=0.01)
-
-  def test_v_max3_f16_basic(self):
-    """V_MAX3_F16: maximum of three f16 values."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(1.0)),
-      s_mov_b32(s[1], f32_to_f16(3.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_max3_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 3.0, delta=0.01)
-
-  def test_v_med3_f16_basic(self):
-    """V_MED3_F16: median of three f16 values."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(3.0)),
-      s_mov_b32(s[1], f32_to_f16(1.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_med3_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 2.0, delta=0.01)
-
-  def test_v_minmax_f16_basic(self):
-    """V_MINMAX_F16: clamp(src0, min=src1, max=src2)."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(2.5)),
-      s_mov_b32(s[1], f32_to_f16(1.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_minmax_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 2.0, delta=0.01)
-
-  def test_v_maxmin_f16_basic(self):
-    """V_MAXMIN_F16: clamp(src0, min=src2, max=src1)."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(0.5)),
-      s_mov_b32(s[1], f32_to_f16(2.0)),
-      s_mov_b32(s[2], f32_to_f16(1.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_maxmin_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 1.0, delta=0.01)
-
-  def test_v_min3_f16_with_neg(self):
-    """V_MIN3_F16 with neg modifier: min(-3, 1, 2) = -3."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(3.0)),
-      s_mov_b32(s[1], f32_to_f16(1.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_min3_f16(v[3], -v[0], v[1], v[2]),  # neg on first operand
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, -3.0, delta=0.01)
-
-  def test_v_max3_f16_with_abs(self):
-    """V_MAX3_F16 with abs modifier: max(|-3|, 1, 2) = 3."""
-    instructions = [
-      s_mov_b32(s[0], f32_to_f16(-3.0)),
-      s_mov_b32(s[1], f32_to_f16(1.0)),
-      s_mov_b32(s[2], f32_to_f16(2.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      v_max3_f16(v[3], abs(v[0]), v[1], v[2]),  # abs on first operand
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 3.0, delta=0.01)
-
-  def test_v_med3_f16_opsel_hi(self):
-    """V_MED3_F16 with opsel reading from hi half."""
-    # Pack two f16 values: hi=5.0, lo=1.0
-    packed = (f32_to_f16(5.0) << 16) | f32_to_f16(1.0)
-    instructions = [
-      s_mov_b32(s[0], packed),
-      s_mov_b32(s[1], f32_to_f16(3.0)),
-      s_mov_b32(s[2], f32_to_f16(4.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], s[2]),
-      # Read hi half of v[0] (5.0), med3(5, 3, 4) = 4
-      v_med3_f16(v[3], v[0].h, v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 4.0, delta=0.01)
-
-
-class TestSadHi(unittest.TestCase):
-  """Tests for V_SAD_HI_U8 instruction."""
-
-  def test_v_sad_hi_u8_basic(self):
-    """V_SAD_HI_U8: (sad << 16) + acc."""
-    # |1-5| + |2-6| + |3-7| + |4-8| = 16, << 16 = 0x100000, + 100 = 0x100064
-    instructions = [
-      v_mov_b32_e32(v[0], 0x04030201),
-      v_mov_b32_e32(v[1], 0x08070605),
-      v_mov_b32_e32(v[2], 100),
-      v_sad_hi_u8(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3], (16 << 16) + 100)
-
-  def test_v_sad_hi_u8_zero_diff(self):
-    """V_SAD_HI_U8: identical inputs gives acc only."""
-    instructions = [
-      v_mov_b32_e32(v[0], 0x12345678),
-      v_mov_b32_e32(v[2], 50),
-      v_sad_hi_u8(v[3], v[0], v[0], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3], 50)
-
-
-class TestPermlane(unittest.TestCase):
-  """Tests for V_PERMLANE16_B32 and V_PERMLANEX16_B32 instructions."""
-
-  def test_v_permlane16_b32_identity(self):
-    """V_PERMLANE16_B32 with identity permutation (lane i reads from lane i within row)."""
-    # lanesel encodes 4 bits per position: position i gets lanesel[i*4+3:i*4]
-    # Identity: position 0->0, 1->1, ..., 15->15
-    # lanesel = 0xFEDCBA9876543210 (positions 15-0 in nibbles)
-    instructions = [
-      v_mov_b32_e32(v[0], 0xDEADBEEF),  # source data
-      s_mov_b32(s[0], 0x76543210),       # lanesel low (positions 0-7)
-      s_mov_b32(s[1], 0xFEDCBA98),       # lanesel high (positions 8-15)
-      v_permlane16_b32(v[1], v[0], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    # Lane 0 reads from lane 0 (position 0 -> lanesel[3:0] = 0)
-    self.assertEqual(st.vgpr[0][1], 0xDEADBEEF)
-
-  def test_v_permlane16_b32_broadcast(self):
-    """V_PERMLANE16_B32 broadcast lane 0 to all lanes in row."""
-    # lanesel = all zeros -> all positions read from lane 0 within row
-    instructions = [
-      v_mov_b32_e32(v[0], 0xCAFEBABE),  # source data
-      s_mov_b32(s[0], 0),                # lanesel low = 0 (all read lane 0)
-      s_mov_b32(s[1], 0),                # lanesel high = 0
-      v_permlane16_b32(v[1], v[0], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=4)
-    # All lanes read from lane 0 of their row
-    for lane in range(4):
-      self.assertEqual(st.vgpr[lane][1], 0xCAFEBABE)
-
-  def test_v_permlanex16_b32_identity(self):
-    """V_PERMLANEX16_B32 cross-row read with identity selection."""
-    # In wave32: row 0 (lanes 0-15) reads from row 1 (lanes 16-31) and vice versa
-    # With single lane in row 0, it reads from lane 0 of row 1 (lane 16)
-    # But lane 16 doesn't exist in 1-lane test, so use 32 lanes
-    instructions = [
-      v_mov_b32_e32(v[0], 0x11111111),  # All lanes have this initially
-      s_mov_b32(s[0], 0x76543210),       # lanesel low
-      s_mov_b32(s[1], 0xFEDCBA98),       # lanesel high
-      v_permlanex16_b32(v[1], v[0], s[0], s[1]),
-    ]
-    st = run_program(instructions, n_lanes=32)
-    # Lane 0 in row 0 reads from lane 0 of row 1 (lane 16)
-    self.assertEqual(st.vgpr[0][1], 0x11111111)
-    # Lane 16 in row 1 reads from lane 0 of row 0 (lane 0)
-    self.assertEqual(st.vgpr[16][1], 0x11111111)
-
-
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vop3p.py

@@ -149,6 +149,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_src2_f16_lo(self):
     """V_FMA_MIX_F32 with src2 as f16 from lo bits."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_2 = f32_to_f16(2.0)
     instructions = [
       s_mov_b32(s[0], f2i(1.0)),
@@ -165,6 +166,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_src2_f16_hi(self):
     """V_FMA_MIX_F32 with src2 as f16 from hi bits."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_2 = f32_to_f16(2.0)
     val = (f16_2 << 16) | 0
     instructions = [
@@ -197,6 +199,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_with_abs_f16_src2_lo(self):
     """V_FMA_MIX_F32 with abs modifier on f16 src2 (lo half). Regression test for sin(1.0) bug."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
     instructions = [
       s_mov_b32(s[0], f2i(0.0)),  # src0 = 0.0 (f32)
@@ -214,6 +217,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_with_neg_f16_src2_lo(self):
     """V_FMA_MIX_F32 with neg modifier on f16 src2 (lo half)."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_1 = f32_to_f16(1.0)  # 0x3c00
     instructions = [
       s_mov_b32(s[0], f2i(0.0)),  # src0 = 0.0 (f32)
@@ -231,6 +235,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_with_abs_f16_src2_hi(self):
     """V_FMA_MIX_F32 with abs modifier on f16 src2 (hi half)."""
+    from extra.assembly.amd.test.hw.helpers import f32_to_f16
     f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
     val = (f16_neg1 << 16) | 0  # -1.0 in hi, 0 in lo
     instructions = [
@@ -249,6 +254,7 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mixlo_f16(self):
     """V_FMA_MIXLO_F16 writes to low 16 bits of destination."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], f2i(2.0)),
       v_mov_b32_e32(v[0], s[0]),
@@ -261,13 +267,14 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = f16(st.vgpr[0][3] & 0xffff)
+    lo = _f16(st.vgpr[0][3] & 0xffff)
     hi = (st.vgpr[0][3] >> 16) & 0xffff
     self.assertAlmostEqual(lo, 7.0, places=1)
     self.assertEqual(hi, 0xdead, f"hi should be preserved, got 0x{hi:04x}")
 
   def test_v_fma_mixlo_f16_all_f32_sources(self):
     """V_FMA_MIXLO_F16 with all f32 sources."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], f2i(1.0)),
       v_mov_b32_e32(v[0], s[0]),
@@ -279,12 +286,13 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = f16(st.vgpr[0][3] & 0xffff)
+    lo = _f16(st.vgpr[0][3] & 0xffff)
     # 1*2+3 = 5
     self.assertAlmostEqual(lo, 5.0, places=1)
 
   def test_v_fma_mixlo_f16_sin_case(self):
     """V_FMA_MIXLO_F16 case from sin kernel."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x3f800000),  # f32 1.0
       v_mov_b32_e32(v[3], s[0]),
@@ -297,7 +305,7 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[3], src1=s[6], src2=v[5], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = f16(st.vgpr[0][3] & 0xffff)
+    lo = _f16(st.vgpr[0][3] & 0xffff)
     self.assertAlmostEqual(lo, -3.14159, delta=0.01)
 
 
@@ -306,6 +314,7 @@ class TestVOP3P(unittest.TestCase):
 
   def test_v_pk_add_f16_basic(self):
     """V_PK_ADD_F16 adds two packed f16 values."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x40003c00),  # hi=2.0, lo=1.0
       s_mov_b32(s[1], 0x44004200),  # hi=4.0, lo=3.0
@@ -315,13 +324,14 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 4.0, places=2)
     self.assertAlmostEqual(hi, 6.0, places=2)
 
   def test_v_pk_mul_f16_basic(self):
     """V_PK_MUL_F16 multiplies two packed f16 values."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x42004000),  # hi=3.0, lo=2.0
       s_mov_b32(s[1], 0x45004400),  # hi=5.0, lo=4.0
@@ -331,13 +341,14 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 8.0, places=1)
     self.assertAlmostEqual(hi, 15.0, places=1)
 
   def test_v_pk_fma_f16_basic(self):
     """V_PK_FMA_F16: D = A * B + C for packed f16."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x42004000),  # A: hi=3.0, lo=2.0
       s_mov_b32(s[1], 0x45004400),  # B: hi=5.0, lo=4.0
@@ -349,8 +360,8 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][3]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 9.0, places=1)   # 2*4+1
     self.assertAlmostEqual(hi, 16.0, places=0)  # 3*5+1
 
@@ -359,6 +370,7 @@ class TestVOP3P(unittest.TestCase):
     Inline constants for VOP3P are f16 values in the low 16 bits only.
     hi half of inline constant is 0, so hi result = v0.hi + 0 = 1.0.
     """
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x3c003c00),  # packed f16: hi=1.0, lo=1.0
       v_mov_b32_e32(v[0], s[0]),
@@ -366,8 +378,8 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     # lo = 1.0 + 1.0 = 2.0, hi = 1.0 + 0.0 = 1.0 (inline const hi half is 0)
     self.assertAlmostEqual(lo, 2.0, places=2)
     self.assertAlmostEqual(hi, 1.0, places=2)
@@ -376,6 +388,7 @@ class TestVOP3P(unittest.TestCase):
     """V_PK_MUL_F16 with inline constant POS_TWO (2.0).
     Inline constant has value only in low 16 bits, hi is 0.
     """
+    from extra.assembly.amd.test.hw.helpers import _f16
     # v0 = packed (3.0, 4.0), multiply by POS_TWO
     # lo = 3.0 * 2.0 = 6.0, hi = 4.0 * 0.0 = 0.0 (inline const hi is 0)
     instructions = [
@@ -385,29 +398,11 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 6.0, places=1)
     self.assertAlmostEqual(hi, 0.0, places=1)
 
-  def test_v_pk_add_u16_float_inline_const_opsel(self):
-    """V_PK_ADD_U16 with float inline constant 2.0
-    Regression test: for integer packed ops, do not perform the f32->f16 conversion.
-    """
-    # src1 = inline float constant 2.0
-    instructions = [
-      s_mov_b32(s[0], 0x00030005),  # packed u16: hi=3, lo=5
-      v_mov_b32_e32(v[0], s[0]),
-      v_pk_add_u16(v[1], v[0], SrcEnum.POS_TWO, opsel_hi=3, opsel_hi2=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][1]
-    lo = result & 0xffff
-    hi = (result >> 16) & 0xffff
-    # lo = 5 + 0x0000 = 0x0005, hi = 3 + 0x4000 = 0x4003
-    self.assertEqual(lo, 0x0005, f"lo: expected 0x0005, got 0x{lo:04x}")
-    self.assertEqual(hi, 0x4003, f"hi: expected 0x4003, got 0x{hi:04x}")
-
 
 class TestWMMAF16(unittest.TestCase):
   """Tests for WMMA F16 output variant (V_WMMA_F16_16X16X16_F16).
@@ -418,6 +413,7 @@ class TestWMMAF16(unittest.TestCase):
 
   def test_v_wmma_f16_16x16x16_f16_all_ones(self):
     """V_WMMA_F16_16X16X16_F16 with all ones produces 16.0 in f16."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = []
     instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
     # Initialize A matrix in v[16:23] (8 regs)
@@ -436,12 +432,13 @@ class TestWMMAF16(unittest.TestCase):
     for lane in range(32):
       for reg in range(8):
         result = st.vgpr[lane][reg]
-        lo = f16(result & 0xffff)
+        lo = _f16(result & 0xffff)
         self.assertAlmostEqual(lo, 16.0, places=1, msg=f"v[{reg}] lane {lane}: expected 16.0, got {lo}")
         self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
 
   def test_v_wmma_f16_16x16x16_f16_with_accumulator(self):
     """V_WMMA_F16_16X16X16_F16 with non-zero accumulator."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = []
     instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
     instructions.append(s_mov_b32(s[1], 0x4500))  # f16 5.0 in lo bits only
@@ -461,7 +458,7 @@ class TestWMMAF16(unittest.TestCase):
     for lane in range(32):
       for reg in range(8):
         result = st.vgpr[lane][reg]
-        lo = f16(result & 0xffff)
+        lo = _f16(result & 0xffff)
         self.assertAlmostEqual(lo, 21.0, places=0, msg=f"v[{reg}] lane {lane}: expected 21.0, got {lo}")
         self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
 
@@ -471,6 +468,7 @@ class TestWMMAF16(unittest.TestCase):
     Regression test: WMMA was using static register indices instead of dynamic.
     This test uses v[64:71] for A, v[80:87] for B, v[96:103] for C/D.
     """
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = []
     instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
     # Initialize A matrix in v[64:71] (8 regs)
@@ -492,7 +490,7 @@ class TestWMMAF16(unittest.TestCase):
     for lane in range(32):
       for reg in range(8):
         result = st.vgpr[lane][reg]
-        lo = f16(result & 0xffff)
+        lo = _f16(result & 0xffff)
         self.assertAlmostEqual(lo, 16.0, places=1, msg=f"v[{reg}] lane {lane}: expected 16.0, got {lo}")
         self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
 
@@ -715,6 +713,7 @@ class TestPackedMixedSigns(unittest.TestCase):
 
   def test_pk_add_f16_mixed_signs(self):
     """V_PK_ADD_F16 with mixed positive/negative values."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0xc0003c00),  # packed: hi=-2.0, lo=1.0
       s_mov_b32(s[1], 0x3c003c00),  # packed: hi=1.0, lo=1.0
@@ -724,13 +723,14 @@ class TestPackedMixedSigns(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
+    lo = _f16(result & 0xffff)
+    hi = _f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 2.0, places=2)   # 1.0 + 1.0
     self.assertAlmostEqual(hi, -1.0, places=2)  # -2.0 + 1.0
 
   def test_pk_mul_f16_zero(self):
     """V_PK_MUL_F16 with zero."""
+    from extra.assembly.amd.test.hw.helpers import _f16
     instructions = [
       s_mov_b32(s[0], 0x40004000),  # packed: 2.0, 2.0
       s_mov_b32(s[1], 0x00000000),  # packed: 0.0, 0.0
@@ -743,277 +743,5 @@ class TestPackedMixedSigns(unittest.TestCase):
     self.assertEqual(result, 0x00000000, "2.0 * 0.0 should be 0.0")
 
 
-class TestDot2F32F16(unittest.TestCase):
-  """Tests for V_DOT2_F32_F16 - dot product of f16 pairs producing f32."""
-
-  def test_v_dot2_f32_f16_basic(self):
-    """V_DOT2_F32_F16: dot product of two packed f16 pairs -> f32."""
-    # src0 = {hi=2.0, lo=1.0}, src1 = {hi=4.0, lo=3.0}
-    # result = 1.0*3.0 + 2.0*4.0 + 0 = 3 + 8 = 11.0
-    src0 = (f32_to_f16(2.0) << 16) | f32_to_f16(1.0)
-    src1 = (f32_to_f16(4.0) << 16) | f32_to_f16(3.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = i2f(st.vgpr[0][3])
-    self.assertAlmostEqual(result, 11.0, places=2)
-
-  def test_v_dot2_f32_f16_with_accumulator(self):
-    """V_DOT2_F32_F16 with non-zero f32 accumulator."""
-    # src0 = {hi=1.0, lo=1.0}, src1 = {hi=1.0, lo=1.0}, acc = 5.0
-    # result = 1.0*1.0 + 1.0*1.0 + 5.0 = 7.0
-    src0 = (f32_to_f16(1.0) << 16) | f32_to_f16(1.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], f2i(5.0)),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[0]),  # same as src0
-      v_mov_b32_e32(v[2], s[1]),
-      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = i2f(st.vgpr[0][3])
-    self.assertAlmostEqual(result, 7.0, places=2)
-
-  def test_v_dot2_f32_f16_negative_values(self):
-    """V_DOT2_F32_F16 with negative f16 values."""
-    # src0 = {hi=-2.0, lo=3.0}, src1 = {hi=1.0, lo=2.0}
-    # result = 3.0*2.0 + (-2.0)*1.0 + 0 = 6 - 2 = 4.0
-    # NOTE: Hardware DOT2 may have up to 1 ULP difference due to internal implementation
-    src0 = (f32_to_f16(-2.0) << 16) | f32_to_f16(3.0)
-    src1 = (f32_to_f16(1.0) << 16) | f32_to_f16(2.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
-    ]
-    st = run_program(instructions, n_lanes=1, ulp_tolerance=1)
-    result = i2f(st.vgpr[0][3])
-    self.assertAlmostEqual(result, 4.0, places=2)
-
-
-class TestDot2F16F16(unittest.TestCase):
-  """Tests for V_DOT2_F16_F16 - dot product of f16 pairs producing f16."""
-
-  def test_v_dot2_f16_f16_basic(self):
-    """V_DOT2_F16_F16: dot product of two packed f16 pairs -> f16."""
-    # src0 = {hi=2.0, lo=1.0}, src1 = {hi=3.0, lo=2.0}
-    # result = 1.0*2.0 + 2.0*3.0 + 0 = 2 + 6 = 8.0 (f16)
-    src0 = (f32_to_f16(2.0) << 16) | f32_to_f16(1.0)
-    src1 = (f32_to_f16(3.0) << 16) | f32_to_f16(2.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot2_f16_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 8.0, places=1)
-
-  def test_v_dot2_f16_f16_with_accumulator(self):
-    """V_DOT2_F16_F16 with non-zero f16 accumulator."""
-    # src0 = {hi=1.0, lo=1.0}, src1 = {hi=1.0, lo=1.0}, acc = 3.0 (f16)
-    # result = 1.0*1.0 + 1.0*1.0 + 3.0 = 5.0 (f16)
-    src0 = (f32_to_f16(1.0) << 16) | f32_to_f16(1.0)
-    acc = f32_to_f16(3.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[2], acc),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[0]),  # same as src0
-      v_mov_b32_e32(v[2], s[2]),
-      v_dot2_f16_f16(v[3], v[0], v[1], v[2]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = f16(st.vgpr[0][3] & 0xffff)
-    self.assertAlmostEqual(result, 5.0, places=1)
-
-
-class TestSignedDotProducts(unittest.TestCase):
-  """Tests for V_DOT4_I32_IU8 and V_DOT8_I32_IU4 with signed inputs."""
-
-  def test_v_dot4_i32_iu8_signed_both(self):
-    """V_DOT4_I32_IU8 with both inputs signed (neg=0b011)."""
-    # src0 = {-1, -2, 3, 4} as i8 = {0xff, 0xfe, 0x03, 0x04}
-    # src1 = {1, 1, 1, 1} as i8
-    # result = (-1)*1 + (-2)*1 + 3*1 + 4*1 = -1 - 2 + 3 + 4 = 4
-    src0 = (0xff << 24) | (0xfe << 16) | (0x03 << 8) | 0x04  # -1, -2, 3, 4
-    src1 = 0x01010101  # 1, 1, 1, 1
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][3]
-    # Result is i32, interpret as signed
-    if result >= 0x80000000:
-      result = result - 0x100000000
-    self.assertEqual(result, 4)
-
-  def test_v_dot4_i32_iu8_src0_signed(self):
-    """V_DOT4_I32_IU8 with only src0 signed (neg=0b001)."""
-    # src0 = {-1, -1, -1, -1} as i8 = {0xff, 0xff, 0xff, 0xff}
-    # src1 = {2, 2, 2, 2} as u8
-    # result = (-1)*2 + (-1)*2 + (-1)*2 + (-1)*2 = -8
-    src0 = 0xffffffff  # -1, -1, -1, -1 (as i8)
-    src1 = 0x02020202  # 2, 2, 2, 2 (as u8)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b001),  # src0 signed
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][3]
-    if result >= 0x80000000:
-      result = result - 0x100000000
-    self.assertEqual(result, -8)
-
-  def test_v_dot4_i32_iu8_src1_signed(self):
-    """V_DOT4_I32_IU8 with only src1 signed (neg=0b010)."""
-    # src0 = {2, 2, 2, 2} as u8
-    # src1 = {-1, -1, -1, -1} as i8 = {0xff, 0xff, 0xff, 0xff}
-    # result = 2*(-1) + 2*(-1) + 2*(-1) + 2*(-1) = -8
-    src0 = 0x02020202  # 2, 2, 2, 2 (as u8)
-    src1 = 0xffffffff  # -1, -1, -1, -1 (as i8)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b010),  # src1 signed
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][3]
-    if result >= 0x80000000:
-      result = result - 0x100000000
-    self.assertEqual(result, -8)
-
-  def test_v_dot4_i32_iu8_unsigned_as_reference(self):
-    """V_DOT4_I32_IU8 with both unsigned (neg=0) - same as V_DOT4_U32_U8."""
-    # src0 = {0xff, 0xff, 0xff, 0xff} = 255 each as u8
-    # src1 = {1, 1, 1, 1}
-    # result = 255*1 + 255*1 + 255*1 + 255*1 = 1020
-    src0 = 0xffffffff
-    src1 = 0x01010101
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0),  # both unsigned
-    ]
-    st = run_program(instructions, n_lanes=1)
-    self.assertEqual(st.vgpr[0][3], 1020)
-
-  def test_v_dot8_i32_iu4_signed_both(self):
-    """V_DOT8_I32_IU4 with both inputs signed (neg=0b011)."""
-    # src0 = 8 nibbles: {-1, -2, 3, 4, -1, -2, 3, 4} as i4
-    # i4 -1 = 0xf, -2 = 0xe, 3 = 0x3, 4 = 0x4
-    # src0 = 0xfe34fe34
-    # src1 = {1, 1, 1, 1, 1, 1, 1, 1} as i4 = 0x11111111
-    # result = 2 * ((-1)*1 + (-2)*1 + 3*1 + 4*1) = 2 * 4 = 8
-    src0 = 0xfe34fe34
-    src1 = 0x11111111
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot8_i32_iu4(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][3]
-    if result >= 0x80000000:
-      result = result - 0x100000000
-    self.assertEqual(result, 8)
-
-  def test_v_dot8_i32_iu4_all_negative(self):
-    """V_DOT8_I32_IU4 with all negative signed values."""
-    # src0 = 8 nibbles all -1 (0xf) = 0xffffffff
-    # src1 = 8 nibbles all 1 = 0x11111111
-    # result = 8 * ((-1)*1) = -8
-    src0 = 0xffffffff  # all -1 as i4
-    src1 = 0x11111111  # all 1
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_mov_b32_e32(v[2], 0),
-      v_dot8_i32_iu4(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][3]
-    if result >= 0x80000000:
-      result = result - 0x100000000
-    self.assertEqual(result, -8)
-
-
-class TestPkMinMaxF16(unittest.TestCase):
-  """Tests for V_PK_MIN_F16 and V_PK_MAX_F16."""
-
-  def test_v_pk_min_f16_basic(self):
-    """V_PK_MIN_F16: packed min of two f16 pairs."""
-    # src0 = {hi=3.0, lo=1.0}, src1 = {hi=2.0, lo=4.0}
-    # result = {min(3,2)=2, min(1,4)=1}
-    src0 = (f32_to_f16(3.0) << 16) | f32_to_f16(1.0)
-    src1 = (f32_to_f16(2.0) << 16) | f32_to_f16(4.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_pk_min_f16(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
-    self.assertAlmostEqual(lo, 1.0, delta=0.01)
-    self.assertAlmostEqual(hi, 2.0, delta=0.01)
-
-  def test_v_pk_max_f16_basic(self):
-    """V_PK_MAX_F16: packed max of two f16 pairs."""
-    # src0 = {hi=3.0, lo=1.0}, src1 = {hi=2.0, lo=4.0}
-    # result = {max(3,2)=3, max(1,4)=4}
-    src0 = (f32_to_f16(3.0) << 16) | f32_to_f16(1.0)
-    src1 = (f32_to_f16(2.0) << 16) | f32_to_f16(4.0)
-    instructions = [
-      s_mov_b32(s[0], src0),
-      s_mov_b32(s[1], src1),
-      v_mov_b32_e32(v[0], s[0]),
-      v_mov_b32_e32(v[1], s[1]),
-      v_pk_max_f16(v[2], v[0], v[1]),
-    ]
-    st = run_program(instructions, n_lanes=1)
-    result = st.vgpr[0][2]
-    lo = f16(result & 0xffff)
-    hi = f16((result >> 16) & 0xffff)
-    self.assertAlmostEqual(lo, 4.0, delta=0.01)
-    self.assertAlmostEqual(hi, 3.0, delta=0.01)
-
-
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/test_emu2_pcode.py

@@ -180,7 +180,7 @@ class TestDSPcodePatterns(unittest.TestCase):
   def test_mem_read_parsing(self):
     """Test MEM[addr].type read expression parsing."""
     # Create a mock LDS buffer
-    lds = UOp(Ops.PARAM, dtypes.uint32.ptr(16384), arg=3)
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
     addr = UOp.const(dtypes.uint32, 0)
     vars = {'_lds': lds, 'ADDR': addr, 'OFFSET': UOp.const(dtypes.uint32, 0)}
 
@@ -213,7 +213,7 @@ class TestDSPcodePatterns(unittest.TestCase):
     """Test DS_LOAD_2ADDR_B32 pcode parsing produces RETURN_DATA assignments."""
     pcode = PCODE.get(DSOp.DS_LOAD_2ADDR_B32)
     self.assertIsNotNone(pcode)
-    lds = UOp(Ops.PARAM, dtypes.uint32.ptr(16384), arg=3)
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
     srcs = {
       'ADDR': UOp.const(dtypes.uint32, 0),
       'OFFSET0': UOp.const(dtypes.uint32, 0),
@@ -286,7 +286,7 @@ class TestAllPcode(unittest.TestCase):
   def _make_srcs(self):
     """Create dummy source variables for pcode parsing."""
     u32, u64 = lambda v=0: UOp.const(dtypes.uint32, v), lambda v=0: UOp.const(dtypes.uint64, v)
-    lds = UOp(Ops.PARAM, dtypes.uint32.ptr(16384), arg=3)
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
     return {'laneId': u32(), 'laneID': u32(), 'S0': u32(), 'S1': u32(), 'S2': u32(), 'S3': u32(), 'SRC0': u32(),
             'D0': u32(), 'D1': u32(), 'DST': u32(), 'VDST': u32(), 'SDST': u32(),
             'VCC': u64(), 'VCCZ': u32(), 'EXEC': u64(), 'EXEC_LO': u32(), 'EXECZ': u32(), 'SCC': u32(),
@@ -294,7 +294,7 @@ class TestAllPcode(unittest.TestCase):
             'ADDR': u32(), 'ADDR_BASE': u32(), 'TADDR': u32(), 'DATA': u32(), 'DATA0': u32(), 'DATA1': u32(), 'DATA2': u32(),
             'VDATA': u32(), 'VDATA0': u32(), 'VDATA1': u32(), 'VDATA2': u32(), 'VDATA3': u32(),
             'OPSEL': u32(), 'OPSEL_HI': u32(), 'NEG': u32(), 'NEG_HI': u32(), 'CLAMP': u32(),
-            'M0': u32(), 'PC': u64(), 'DENORM': u32(1), 'ROUND_MODE': u32(), 'ROUND_TOWARD_ZERO': u32(), 'ROUND_NEAREST_EVEN': u32(), 'WAVE_STATUS': u32(),
+            'M0': u32(), 'PC': u64(), 'DENORM': u32(1), 'ROUND_MODE': u32(), 'WAVE_STATUS': u32(),
             'MAX_FLOAT_F32': u32(0x7f7fffff), 'Unsigned': u32(1), 'clampedLOD': u32(),
             '_lds': lds, '_vmem': lds, '_active': UOp.const(dtypes.bool, True)}
 

extra/assembly/amd/test/test_rdna4_emu.py

@@ -1,98 +0,0 @@
-import unittest, ctypes
-from extra.assembly.amd.autogen.rdna4 import ins as ir4
-from extra.assembly.amd.dsl import v, s
-from extra.assembly.amd.emu import WaveState, decode_program
-from tinygrad.device import Buffer, BufferSpec
-from tinygrad.dtype import dtypes
-
-class TestRDNA4Emu(unittest.TestCase):
-  def _run(self, insts: list, sgprs: dict[int, int] = None, vgprs: dict[tuple[int, int], int] = None) -> WaveState:
-    """Run instructions and return final WaveState."""
-    # Add S_ENDPGM if not present
-    if not any(isinstance(i, ir4.SOPP) and i.op == ir4.SOPPOp.S_ENDPGM for i in insts):
-      insts = list(insts) + [ir4.SOPP(ir4.SOPPOp.S_ENDPGM, simm=0)]
-
-    # Assemble and decode
-    code = b''.join(i.to_bytes() for i in insts)
-    code_buf = (ctypes.c_uint8 * len(code)).from_buffer_copy(code)
-    code_addr = ctypes.addressof(code_buf)
-    program_raw = decode_program(code, "rdna4")
-    program = {code_addr + offset: val for offset, val in program_raw.items()}
-
-    # Setup wave state
-    st = WaveState(n_lanes=1)
-    st.pc = code_addr
-    if sgprs:
-      for idx, val in sgprs.items(): st._write_sgpr(idx, val)
-    if vgprs:
-      for (reg, lane), val in vgprs.items(): st._write_vgpr(reg, lane, val)
-
-    # Setup vmem buffer with external_ptr=0 (maps to address 0, allows any pointer access)
-    vmem_buf = Buffer('CPU', 1 << 40, dtypes.uint32, options=BufferSpec(external_ptr=0)).ensure_allocated()
-
-    # Execute
-    c_bufs = [ctypes.c_uint64(st.sgpr_buf._buf.va_addr), ctypes.c_uint64(st.vgpr_buf._buf.va_addr),
-              ctypes.c_uint64(vmem_buf._buf.va_addr), ctypes.c_uint64(0), ctypes.c_uint64(0)]
-    for _ in range(100):
-      if (pc := st.pc) == 0xFFFFFFFFFFFFFFFF or pc not in program: break
-      _, fxn, globals_list, _ = program[pc]
-      fxn(*[c_bufs[g] for g in globals_list])
-    return st
-
-  def test_vopd_dual_mov(self):
-    """Test VOPD with two V_DUAL_MOV_B32 operations: v[1]=s[1], v[2]=s[2]."""
-    insts = [ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-                      vdstx=v[1], vdsty=v[2], srcx0=s[1], srcy0=s[2], vsrcx1=v[0], vsrcy1=v[0])]
-    st = self._run(insts, sgprs={1: 0x40e00000, 2: 0x41100000})  # 7.0f, 9.0f
-    self.assertEqual(st._read_vgpr(1, 0), 0x40e00000)  # v[1] = 7.0
-    self.assertEqual(st._read_vgpr(2, 0), 0x41100000)  # v[2] = 9.0
-
-  def test_vopd_dual_mov_after_other_vopd(self):
-    """Test VOPD reuse: first VOPD(v[3]=0, v[0]=?), then VOPD(v[1]=s[1], v[2]=s[2])."""
-    # This matches the BEAM kernel sequence that fails
-    insts = [
-      ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-               vdstx=v[3], vdsty=v[0], srcx0=0, srcy0=s[0], vsrcx1=v[0], vsrcy1=v[0]),  # v[3]=0, v[0]=s[0]
-      ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-               vdstx=v[1], vdsty=v[2], srcx0=s[1], srcy0=s[2], vsrcx1=v[0], vsrcy1=v[0]),  # v[1]=s[1], v[2]=s[2]
-    ]
-    st = self._run(insts, sgprs={0: 0x40a00000, 1: 0x40e00000, 2: 0x41100000})  # 5.0f, 7.0f, 9.0f
-    self.assertEqual(st._read_vgpr(1, 0), 0x40e00000)  # v[1] = 7.0
-    self.assertEqual(st._read_vgpr(2, 0), 0x41100000)  # v[2] = 9.0
-
-  def test_vopd_with_s_add_f32_sequence(self):
-    """Test full BEAM kernel sequence: s_add_f32 then VOPD."""
-    # This is the exact sequence from the failing BEAM kernel
-    insts = [
-      ir4.SOP2(ir4.SOP2Op.S_ADD_F32, sdst=s[0], ssrc0=s[0], ssrc1=s[8]),   # s[0] = s[0] + s[8]
-      ir4.SOP2(ir4.SOP2Op.S_ADD_F32, sdst=s[1], ssrc0=s[1], ssrc1=s[9]),   # s[1] = s[1] + s[9]
-      ir4.SOP2(ir4.SOP2Op.S_ADD_F32, sdst=s[2], ssrc0=s[2], ssrc1=s[10]),  # s[2] = s[2] + s[10]
-      ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-               vdstx=v[3], vdsty=v[0], srcx0=0, srcy0=s[0], vsrcx1=v[0], vsrcy1=v[0]),
-      ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-               vdstx=v[1], vdsty=v[2], srcx0=s[1], srcy0=s[2], vsrcx1=v[0], vsrcy1=v[0]),
-    ]
-    # Input: s[0:2] = [1,2,3], s[8:10] = [4,5,6]
-    # After s_add_f32: s[0:2] = [5,7,9]
-    st = self._run(insts, sgprs={0: 0x3f800000, 1: 0x40000000, 2: 0x40400000,  # 1.0, 2.0, 3.0
-                                  8: 0x40800000, 9: 0x40a00000, 10: 0x40c00000})  # 4.0, 5.0, 6.0
-    self.assertEqual(st._read_vgpr(1, 0), 0x40e00000)  # v[1] = 7.0
-    self.assertEqual(st._read_vgpr(2, 0), 0x41100000)  # v[2] = 9.0
-
-  def test_s_mov_b32_then_vopd(self):
-    """Test s_mov_b32 followed by VOPD - simulates BEAM kernel sequence."""
-    # Use s_mov_b32 with SGPR source (copy from pre-initialized SGPRs)
-    # s[10:12] will have values set by test harness, copy to s[0:2], then VOPD to VGPRs
-    insts = [
-      ir4.SOP1(ir4.SOP1Op.S_MOV_B32, sdst=s[0], ssrc0=s[10]),  # s[0] = s[10]
-      ir4.SOP1(ir4.SOP1Op.S_MOV_B32, sdst=s[1], ssrc0=s[11]),  # s[1] = s[11]
-      ir4.SOP1(ir4.SOP1Op.S_MOV_B32, sdst=s[2], ssrc0=s[12]),  # s[2] = s[12]
-      ir4.VOPD(ir4.VOPDOp.V_DUAL_MOV_B32, ir4.VOPDOp.V_DUAL_MOV_B32,
-               vdstx=v[1], vdsty=v[2], srcx0=s[1], srcy0=s[2], vsrcx1=v[0], vsrcy1=v[0]),
-    ]
-    st = self._run(insts, sgprs={10: 0x40a00000, 11: 0x40e00000, 12: 0x41100000})  # 5.0, 7.0, 9.0
-    self.assertEqual(st._read_vgpr(1, 0), 0x40e00000)  # v[1] = 7.0
-    self.assertEqual(st._read_vgpr(2, 0), 0x41100000)  # v[2] = 9.0
-
-if __name__ == '__main__':
-  unittest.main()

extra/assembly/amd/test/test_sqtt_examples.py

@@ -203,12 +203,12 @@ class SQTTExamplesTestBase(unittest.TestCase):
 class TestSQTTExamplesRDNA3(SQTTExamplesTestBase):
   target = "gfx1100"
   expected = {
-    "profile_empty_run_0": [1844, 1885, 1905, 1956, 1983, 1889],
-    "profile_empty_run_1": [1780, 1885, 1905, 1956, 1983, 1889],
-    "profile_gemm_run_0": [2656, 2025, 2045, 2096, 2123, 2029, 3183, 2019, 2039, 2090, 2117, 2023, 19119, 2013, 2033, 2084, 2111, 2017],
-    "profile_gemm_run_1": [2662, 2025, 2045, 2096, 2123, 2029, 3179, 2019, 2039, 2090, 2117, 2023, 19113, 2071, 2091, 2142, 2169, 2075],
-    "profile_plus_run_0": [1886, 2013, 2033, 2084, 2111, 2017],
-    "profile_plus_run_1": [1988, 2071, 2091, 2142, 2169, 2075],
+    "profile_empty_run_0": [1803, 1908, 1928, 1979, 2006, 1912],
+    "profile_empty_run_1": [1803, 1908, 1928, 1979, 2006, 1912],
+    "profile_gemm_run_0": [2531, 1844, 1864, 1915, 1942, 1848, 3074, 1919, 1939, 1990, 2017, 1923, 19026, 1919, 1939, 1990, 2017, 1929],
+    "profile_gemm_run_1": [2554, 1844, 1864, 1915, 1942, 1848, 3084, 1919, 1939, 1990, 2017, 1923, 19010, 1919, 1939, 1990, 2017, 1923],
+    "profile_plus_run_0": [1900, 1908, 1928, 1979, 2006, 1912],
+    "profile_plus_run_1": [1856, 1908, 1928, 1979, 2006, 1912],
   }
 
 class TestSQTTExamplesRDNA4(SQTTExamplesTestBase): target = "gfx1200"

extra/gemm/amd_asm_matmul.py

@@ -471,7 +471,7 @@ THREADS = 128
 
 def test_matmul():
   dev = Device[Device.DEFAULT]
-  print(f"Device arch: {dev.renderer.arch}")
+  print(f"Device arch: {dev.arch}")
 
   if getenv("STOCK", 0):
     # Load the stock kernel from amd_seb/kernel8_batched_gmem.s
@@ -479,7 +479,7 @@ def test_matmul():
     asm = stock_path.read_text()
     print(f"Loaded stock kernel from {stock_path}")
   else:
-    asm = build_kernel(dev.renderer.arch)
+    asm = build_kernel(dev.arch)
 
   binary = dev.compiler.compile(asm)
   print(f"Compiled! Binary size: {len(binary)} bytes")

extra/gemm/asm/cdna/gemm.py

@@ -1,96 +0,0 @@
-import atexit, functools
-from tinygrad.runtime.support.compiler_amd import HIPCompiler
-from tinygrad import Tensor, Device, dtypes
-from tinygrad.uop.ops import UOp, Ops, KernelInfo, AxisType
-from tinygrad.renderer import Estimates
-from tinygrad.helpers import getenv, all_same, dedup
-from extra.gemm.asm.cdna.asm import build_kernel, GEMM_ARGS
-
-# ** CDNA4 assembly gemm
-
-WORKGROUP_SIZE = 256
-
-def custom_asm_gemm(C:UOp, A:UOp, B:UOp, dname:str, arch:str, wg:int) -> UOp:
-  batch, M, K = A.shape
-  K2, N = B.shape[(1 if B.ndim == 3 else 0):]
-  assert K == K2
-  lidx = UOp.special(WORKGROUP_SIZE, "lidx0")
-  gidx = UOp.special(wg, "gidx0")
-  k = build_kernel(batch, M, N, K, A.dtype.base)
-  sink = UOp.sink(C.base, A.base, B.base, lidx, gidx,
-                  arg=KernelInfo(name=k.name, estimates=Estimates(ops=2*batch*M*N*K, mem=(batch*M*K + K*N + batch*M*N)*2)))
-  binary = HIPCompiler(arch).compile(k.to_asm())
-  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
-                               UOp(Ops.SOURCE, arg=k.to_text()), UOp(Ops.BINARY, arg=binary)))
-
-counters = {"used":0, "todos":[]}
-def todo(msg:str) -> bool: counters["todos"].append(msg); return False
-atexit.register(lambda: print(f'asm_gemm: {counters["used"]} used, {len(counters["todos"])} not used'))
-
-def can_use_asm_gemm(a:Tensor, b:Tensor) -> bool:
-  if a.dtype != b.dtype: return todo(f"dtypes must match {a.dtype} != {b.dtype}")
-  if a.dtype not in {dtypes.bfloat16, dtypes.float16}: return todo(f"only bfloat16/float16, got {a.dtype}")
-  # only sharding on the batch is tested, others might work too
-  if isinstance(a.device, tuple) and not (a.ndim == 3 and a.uop.axis == 0 and b.uop.axis is None):
-    return todo(f"sharding mismatch a.ndim={a.ndim} a.uop.axis={a.uop.axis} b.uop.axis={b.uop.axis}")
-  batch, M, K = (1, *a.shape) if a.ndim == 2 else a.shape
-  N = b.shape[1]
-  if isinstance(a.device, tuple):
-    batch //= len(a.device)
-    dname = a.device[0]
-  else: dname = a.device
-  arch = getattr(Device[dname].renderer, "arch", "")
-  if batch not in {1, 2}: return todo(f"GEMM batch size {batch}")
-  if (key:=(M, N, K)) not in GEMM_ARGS and arch == "gfx950": return todo(f"GEMM shape not supported {key} on {arch}")
-  return True
-
-# ** UOp gemm to test Tensor.custom_kernel multi and backward correctness on non cdna4
-# note: this can be removed after we have GEMM on mixins
-
-def custom_uop_gemm(C:UOp, A:UOp, B:UOp) -> UOp:
-  M, K = A.shape[0]*A.shape[1], A.shape[2]
-  K2, N = B.shape[(1 if B.ndim == 3 else 0):]
-  assert K == K2
-  m = UOp.range(M, 1, AxisType.LOOP)
-  n = UOp.range(N, 2, AxisType.LOOP)
-  k = UOp.range(K, 0, AxisType.REDUCE)
-  mul = (A.index((m*UOp.const(dtypes.index, K)+k))*B.index((k*UOp.const(dtypes.index, N)+n))).cast(dtypes.float32)
-  red = mul.reduce(k, arg=Ops.ADD, dtype=dtypes.float32).cast(C.dtype.base)
-  store = C.index((m*UOp.const(dtypes.index, N)+n), ptr=True).store(red).end(m, n)
-  return store.sink(arg=KernelInfo(name=f'uop_gemm_{M}_{N}_{K}'))
-
-# ** backward gemm, might use the asm gemm
-
-def custom_gemm_bw(gradient:UOp, kernel:UOp):
-  out, a, b = kernel.src
-  assert all_same([gradient.device, a.device, b.device, out.device])
-  a_t, b_t, g_t = Tensor(a, device=a.device), Tensor(b, device=a.device), Tensor(gradient, device=a.device)
-  grad_a = (g_t @ b_t.T).uop
-  grad_b = (a_t.permute(2, 0, 1).reshape(a_t.shape[2], -1) @ g_t.reshape(-1, g_t.shape[-1])).uop
-  return (None, grad_a, grad_b)
-
-# ** main gemm function
-
-def asm_gemm(a:Tensor, b:Tensor) -> Tensor:
-  assert can_use_asm_gemm(a, b), f"{counters['todos'][-1]}"
-  counters["used"] += 1
-  squeeze = a.ndim == 2
-  if squeeze: a = a.unsqueeze(0)
-
-  batch, M, K = a.shape
-  N = b.shape[1]
-  is_multi = isinstance(a.device, tuple)
-
-  if is_multi:
-    out = Tensor(Tensor.empty(batch//len(a.device), M, N, dtype=a.dtype, device=a.device).uop.multi(0), device=a.device)
-  else:
-    out = Tensor.empty(batch, M, N, dtype=a.dtype, device=a.device)
-
-  dname = a.device[0] if is_multi else a.device
-  arch = getattr(Device[dname].renderer, "arch", "")
-  if arch.startswith("gfx950") and getenv("USE_ASM", 1):
-    numWG = GEMM_ARGS[(M, N, K)][0]
-    out = Tensor.custom_kernel(out, a, b, fxn=functools.partial(custom_asm_gemm, dname=dname, wg=numWG, arch=arch), grad_fxn=custom_gemm_bw)[0]
-  else:
-    out = Tensor.custom_kernel(out, a, b, fxn=custom_uop_gemm, grad_fxn=custom_gemm_bw)[0]
-  return out.squeeze(0) if squeeze else out

extra/gemm/asm/cdna/template.s

@@ -0,0 +1,78 @@
+.text
+.section	.text.
+.global	gemm
+.p2align	8
+.type	gemm,@function
+
+gemm:
+INSTRUCTIONS
+
+.section .rodata,"a",@progbits
+.p2align 6, 0x0
+.amdhsa_kernel gemm
+  # basic memory requirements
+  .amdhsa_group_segment_fixed_size 133120
+  .amdhsa_private_segment_fixed_size 0
+  .amdhsa_kernarg_size 28
+  # register usage (RSRC1)
+  .amdhsa_next_free_vgpr 504
+  .amdhsa_next_free_sgpr 96
+  # workgroup / workitem IDs (RSRC2)
+  .amdhsa_system_sgpr_workgroup_id_x 1
+  .amdhsa_system_sgpr_workgroup_id_y 1
+  .amdhsa_system_sgpr_workgroup_id_z 1
+  # user SGPRs, we only specify the kernel args ptr in s[0:1]
+  .amdhsa_user_sgpr_kernarg_segment_ptr 1
+  .amdhsa_user_sgpr_count 2
+  .amdhsa_user_sgpr_kernarg_preload_length 0
+  .amdhsa_user_sgpr_kernarg_preload_offset 0
+  # gfx90a / gfx940 specifics (RSRC3)
+  .amdhsa_accum_offset 248
+  .amdhsa_uses_dynamic_stack 0
+  .amdhsa_tg_split 0
+.end_amdhsa_kernel
+
+.amdgpu_metadata
+---
+amdhsa.kernels:
+  - .name: gemm
+    .symbol: gemm.kd
+    .args:
+      - .name: C
+        .address_space: global
+        .offset: 0
+        .size: 8
+        .value_kind: global_buffer
+        .value_type: bf16
+      - .name: B
+        .address_space: global
+        .offset: 8
+        .size: 8
+        .value_kind: global_buffer
+        .value_type: bf16
+      - .name: A
+        .address_space: global
+        .offset: 16
+        .size: 8
+        .value_kind: global_buffer
+        .value_type: bf16
+      - .name: sz
+        .offset: 24
+        .size: 4
+        .value_kind: by_value
+        .value_type: u32
+    .group_segment_fixed_size: 133120
+    .private_segment_fixed_size: 0
+    .kernarg_segment_align: 8
+    .kernarg_segment_size: 28
+    .max_flat_workgroup_size: 256
+    .sgpr_count: 88
+    .sgpr_spill_count: 0
+    .vgpr_count: 248
+    .vgpr_spill_count: 0
+    .wavefront_size: 64
+amdhsa.version:
+  - 1
+  - 0
+...
+.end_amdgpu_metadata

extra/gemm/asm/cdna/test.py

@@ -0,0 +1,73 @@
+# Run assembly on the AMD runtime and check correctness
+# VIZ=2 to profile
+import pathlib
+from tinygrad import Tensor, Device, dtypes, Context
+from tinygrad.uop.ops import UOp, Ops, KernelInfo
+from tinygrad.engine.realize import Estimates
+from tinygrad.helpers import getenv
+
+fp = pathlib.Path(__file__).parent/"gemm.s"
+
+N = getenv("N", 8192)
+THREADS_PER_WG = 256
+NUM_WG = N//THREADS_PER_WG * N//THREADS_PER_WG
+
+assert N % THREADS_PER_WG == 0, "N must be divisible by THREADS_PER_WG"
+
+# ** generate inputs on CPU
+
+scale = 10.0
+
+import torch
+torch.manual_seed(0)
+A = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
+B = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
+Bt = B.t().contiguous() # transpose B for the asm gemm
+C_torch = A@B
+
+# ** copy buffers to AMD
+
+# input creation and validation run on the copy engine for simpler tracing
+
+def from_torch(t:torch.Tensor) -> Tensor:
+  return Tensor.from_blob(t.data_ptr(), t.shape, dtype=dtypes.bfloat16, device="cpu").to(Device.DEFAULT).realize()
+
+C_tiny = from_torch(A) @ from_torch(B)
+C_asm = Tensor.empty_like(C_tiny)
+
+# ** assembly custom kernel
+
+def custom_asm_gemm(C:UOp, A:UOp, B:UOp) -> UOp:
+  lidx = UOp.special(THREADS_PER_WG, "lidx0")
+  gidx = UOp.special(NUM_WG, "gidx0")
+
+  src = (pathlib.Path(__file__).parent/"template.s").read_text().replace("INSTRUCTIONS", fp.read_text())
+
+  sz = UOp.variable("SZ", 256, 8192)
+
+  sink = UOp.sink(C.base, A.base, B.base, sz, lidx, gidx, arg=KernelInfo(name="gemm", estimates=Estimates(ops=N*N*N*2, mem=N*N*4*3)))
+  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=Device.DEFAULT), UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=src)))
+
+C_asm = Tensor.custom_kernel(C_asm, from_torch(A), from_torch(Bt), fxn=custom_asm_gemm)[0]
+
+# ** run gemms
+
+sched = Tensor.schedule(C_tiny, C_asm)
+eis = [si.lower() for si in sched]
+
+with Context(DEBUG=2):
+  for ei in eis:
+    et = ei.run({"SZ":N}, wait=True)
+    print(f"{(N*N*N*2 / et)*1e-12:.2f} REAL TFLOPS")
+
+# ** correctness
+
+import ctypes
+
+def torch_bf16(t:Tensor) -> torch.tensor:
+  asm_out = t.to("cpu").realize().uop.buffer._buf
+  buf = (ctypes.c_uint16*C_asm.uop.size).from_address(asm_out.va_addr)
+  return torch.frombuffer(buf, dtype=torch.bfloat16, count=C_asm.uop.size).reshape(C_asm.shape)
+
+assert torch.allclose(torch_bf16(C_asm), C_torch, rtol=1e-2, atol=1e-3)
+assert torch.allclose(torch_bf16(C_tiny), C_torch, rtol=1e-2, atol=1e-3)

extra/gemm/intel_xmx.py

@@ -37,7 +37,7 @@ b.copyin(row.data)
 c.copyin(mat.data)
 ret = prog(a._buf, b._buf, c._buf, global_size=[1,1,1], local_size=[8,1,1], wait=True)
 print(ret)
-out = np.frombuffer(a.as_memoryview(), np.float32)
+out = np.frombuffer(a.as_buffer(), np.float32)
 real = row.astype(np.float32)@mat.T.astype(np.float32)
 print("out:", out)
 print("real", real)

extra/gemm/triton_nv_matmul.py

@@ -98,10 +98,10 @@ if __name__ == "__main__":
   # check correctness
   if getenv("VERIFY"):
     from tinygrad.engine.realize import run_schedule
-    triton_buf = np.frombuffer(si.bufs[0].as_memoryview(), np.float16).reshape(M,N)
+    triton_buf = np.frombuffer(si.bufs[0].as_buffer(), np.float16).reshape(M,N)
     print(triton_buf)
     run_schedule(sched)
-    tinygrad_buf = np.frombuffer(si.bufs[0].as_memoryview(), np.float16).reshape(M,N)
+    tinygrad_buf = np.frombuffer(si.bufs[0].as_buffer(), np.float16).reshape(M,N)
     print(tinygrad_buf)
     np.testing.assert_allclose(triton_buf, tinygrad_buf)
     print("correct!")

extra/hip_large_kernel.py

@@ -18,7 +18,7 @@ prg = dev.runtime("write_ones", mbin)
 prg(buf0._buf, global_size=(1,65537,1), local_size=(1,1,1), wait=True)
 
 import numpy as np
-def to_np(buf): return np.frombuffer(buf.as_memoryview().cast(buf.dtype.base.fmt), dtype=_to_np_dtype(buf.dtype.base))
+def to_np(buf): return np.frombuffer(buf.as_buffer().cast(buf.dtype.base.fmt), dtype=_to_np_dtype(buf.dtype.base))
 
 big = to_np(buf0)
 print(big)

extra/huggingface_onnx/huggingface_manager.py

@@ -8,14 +8,14 @@ from tinygrad.helpers import _ensure_downloads_dir
 DOWNLOADS_DIR = _ensure_downloads_dir() / "models"
 from tinygrad.helpers import tqdm
 
-def snapshot_download_with_retry(*, repo_id: str, allow_patterns: list[str]|tuple[str, ...]|None=None, local_dir: str|Path|None=None,
+def snapshot_download_with_retry(*, repo_id: str, allow_patterns: list[str]|tuple[str, ...]|None=None, cache_dir: str|Path|None=None,
                                  tries: int=2, **kwargs) -> Path:
   for attempt in range(tries):
     try:
       return Path(snapshot_download(
         repo_id=repo_id,
         allow_patterns=allow_patterns,
-        local_dir=str(local_dir) if local_dir is not None else None,
+        cache_dir=str(cache_dir) if cache_dir is not None else None,
         **kwargs
       ))
     except Exception as e:
@@ -144,14 +144,14 @@ class HuggingFaceONNXManager:
         root_path = snapshot_download_with_retry(
           repo_id=model_id,
           allow_patterns=allow_patterns,
-          local_dir=str(self.models_dir / model_id)
+          cache_dir=str(self.models_dir)
         )
 
         # Download config files (usually small)
         snapshot_download_with_retry(
           repo_id=model_id,
           allow_patterns=["*config.json"],
-          local_dir=str(self.models_dir / model_id)
+          cache_dir=str(self.models_dir)
         )
 
         model_data["download_path"] = str(root_path)

extra/huggingface_onnx/run_models.py

@@ -88,8 +88,8 @@ if __name__ == "__main__":
       # repo id
       # validates all onnx models inside repo
       repo_id = "/".join(path)
-      root_path = snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*.onnx", "*.onnx_data"], local_dir=DOWNLOADS_DIR / repo_id)
-      snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*config.json"], local_dir=DOWNLOADS_DIR / repo_id)
+      root_path = snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*.onnx", "*.onnx_data"], cache_dir=DOWNLOADS_DIR)
+      snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*config.json"], cache_dir=DOWNLOADS_DIR)
       config = get_config(root_path)
       for onnx_model in root_path.rglob("*.onnx"):
         rtol, atol = get_tolerances(onnx_model.name)
@@ -101,8 +101,8 @@ if __name__ == "__main__":
       onnx_model = path[-1]
       assert path[-1].endswith(".onnx")
       repo_id, relative_path = "/".join(path[:2]), "/".join(path[2:])
-      root_path = snapshot_download_with_retry(repo_id=repo_id, allow_patterns=[relative_path], local_dir=DOWNLOADS_DIR / repo_id)
-      snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*config.json"], local_dir=DOWNLOADS_DIR / repo_id)
+      root_path = snapshot_download_with_retry(repo_id=repo_id, allow_patterns=[relative_path], cache_dir=DOWNLOADS_DIR)
+      snapshot_download_with_retry(repo_id=repo_id, allow_patterns=["*config.json"], cache_dir=DOWNLOADS_DIR)
       config = get_config(root_path)
       rtol, atol = get_tolerances(onnx_model)
       print(f"validating {relative_path} with truncate={args.truncate}, {rtol=}, {atol=}")

extra/models/clip.py

@@ -5,6 +5,7 @@ from tinygrad.nn import Linear, LayerNorm, Embedding, Conv2d
 from typing import List, Optional, Union, Tuple, Dict
 from abc import ABC, abstractmethod
 from functools import lru_cache
+from PIL import Image
 import numpy as np
 import re, gzip
 
@@ -443,8 +444,7 @@ class OpenClipEncoder:
   # TODO:
   # Should be doable in pure tinygrad, would just require some work and verification.
   # This is very desirable since it would allow for full generation->evaluation in a single JIT call.
-  def prepare_image(self, image) -> Tensor:
-    from PIL import Image
+  def prepare_image(self, image:Image.Image) -> Tensor:
     SIZE = 224
     w, h = image.size
     scale = min(SIZE / h, SIZE / w)

extra/models/llama.py

@@ -61,9 +61,6 @@ class Attention:
       xq = self.q_norm(xq)
       xk = self.k_norm(xk)
 
-    # cast_float_to_bf16 is expensive in reduction loops, break it out
-    if x.dtype == dtypes.bfloat16: xq, xk = xq.contiguous_backward(), xk.contiguous_backward()
-
     xq = xq.reshape(xq.shape[0], xq.shape[1], self.n_heads, self.head_dim)
     xk = xk.reshape(xk.shape[0], xk.shape[1], self.n_kv_heads, self.head_dim)
     xv = xv.reshape(xv.shape[0], xv.shape[1], self.n_kv_heads, self.head_dim)
@@ -89,14 +86,11 @@ class Attention:
       assert start_pos == 0
       keys, values = xk, xv
 
-    if Tensor.training:
-      xq, keys, values = xq.transpose(1, 2), keys.transpose(1, 2), values.transpose(1, 2)
-      attn = xq.scaled_dot_product_attention(keys, values, is_causal=True, enable_gqa=True).transpose(1, 2)
-    else:
-      keys, values = repeat_kv(keys, self.n_rep), repeat_kv(values, self.n_rep)
-      xq, keys, values = xq.transpose(1, 2), keys.transpose(1, 2), values.transpose(1, 2)
-      attn = xq.scaled_dot_product_attention(keys, values, mask).transpose(1, 2)
+    keys, values = repeat_kv(keys, self.n_rep), repeat_kv(values, self.n_rep)
+    xq, keys, values = xq.transpose(1, 2), keys.transpose(1, 2), values.transpose(1, 2)
+    attn = xq.scaled_dot_product_attention(keys, values, mask).transpose(1, 2)
     if getenv("STUB_ATTENTION"):
+      # TODO: do we need mask?
       from tinygrad.uop.ops import UOp, KernelInfo
       def fa_custom_forward(attn:UOp, q:UOp, k:UOp, v:UOp) -> UOp:
         return UOp.sink(arg=KernelInfo(name="fa_custom_forward"))
@@ -203,9 +197,7 @@ class Transformer:
     h = self.tok_embeddings(tokens)
     freqs_cis = self.freqs_cis.cast(h.dtype)[:, start_pos:start_pos+seqlen, :, :, :]
 
-    if not Tensor.training and seqlen > 1:
-      mask = Tensor.full((1, 1, seqlen, start_pos+seqlen), float("-inf"), dtype=h.dtype, device=h.device).triu(start_pos+1)
-    else: mask = None
+    mask = Tensor.full((1, 1, seqlen, start_pos+seqlen), float("-inf"), dtype=h.dtype, device=h.device).triu(start_pos+1) if seqlen > 1 else None
     for layer in self.layers: h = layer(h, start_pos, freqs_cis, mask)
     logits = self.output(self.norm(h))
     if math.isnan(temperature): return logits

extra/optimization/generate_dataset.sh

@@ -7,7 +7,7 @@ export CAPTURE_PROCESS_REPLAY=1
 rm "$LOGOPS" 2>/dev/null || true
 test/external/process_replay/reset.py
 
-CI=1 python3 -m pytest -n=auto test/test_ops.py test/test_nn.py test/unit/test_winograd.py test/null/test_real_world.py --durations=20
+CI=1 python3 -m pytest -n=auto test/test_ops.py test/test_nn.py test/unit/test_winograd.py test/models/test_real_world.py --durations=20
 CL=1 python3 -m pytest test/test_tiny.py
 
 # extract, sort and uniq

extra/optimization/get_action_space.py

@@ -0,0 +1,31 @@
+import random
+from extra.optimization.helpers import load_worlds, ast_str_to_lin
+from tinygrad.codegen.opt.search import actions
+from tinygrad.codegen.opt.kernel import Kernel
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from tinygrad.helpers import tqdm
+
+tactions = set()
+def test_rebuild(lin):
+  linr = Kernel(lin.ast)
+  for o in lin.applied_opts:
+    assert o in actions, f"{o} is not in actions"
+    tactions.add(o)
+    linr.apply_opt(o)
+
+  assert len(lin.sts) == len(linr.sts)
+  for st1,st2 in zip(lin.sts, linr.sts):
+    assert st1 == st2, f"{st1} != {st2}"
+
+if __name__ == "__main__":
+  ast_strs = load_worlds(False, False, False)
+  random.shuffle(ast_strs)
+  ast_strs = ast_strs[:2000]
+  for ast_str in tqdm(ast_strs):
+    lin = ast_str_to_lin(ast_str)
+    #if not lin.apply_tensor_cores():
+    lin.apply_opts(hand_coded_optimizations(lin))
+    test_rebuild(lin)
+
+  print(len(tactions), len(actions))
+  print(sorted(list(tactions)))

extra/optimization/rl.py

@@ -0,0 +1,76 @@
+import os
+import numpy as np
+import math, random
+from tinygrad.tensor import Tensor
+from tinygrad.nn.state import get_parameters, get_state_dict, safe_save, safe_load, load_state_dict
+from tinygrad.codegen.opt.search import actions, bufs_from_lin, get_kernel_actions
+from tinygrad.nn.optim import Adam
+from extra.optimization.extract_policynet import PolicyNet
+from extra.optimization.helpers import load_worlds, ast_str_to_lin, lin_to_feats, time_linearizer
+
+if __name__ == "__main__":
+  net = PolicyNet()
+  if os.path.isfile("/tmp/policynet.safetensors"): load_state_dict(net, safe_load("/tmp/policynet.safetensors"))
+  optim = Adam(get_parameters(net))
+
+  ast_strs = load_worlds()
+
+  # select a world
+  all_feats, all_acts, all_rews = [], [], []
+  while 1:
+    Tensor.training = False
+    lin = ast_str_to_lin(random.choice(ast_strs))
+    rawbufs = bufs_from_lin(lin)
+    tm = last_tm = base_tm = time_linearizer(lin, rawbufs)
+
+    # take actions
+    feats, acts, rews = [], [], []
+    while 1:
+      feat = lin_to_feats(lin)
+      feats.append(feat)
+      probs = net(Tensor([feat])).exp()[0].numpy()
+
+      # mask valid actions
+      valid_action_mask = np.zeros((len(actions)+1), dtype=np.float32)
+      for x in get_kernel_actions(lin): valid_action_mask[x] = 1
+      probs *= valid_action_mask
+      probs /= sum(probs)
+
+      act = np.random.choice(len(probs), p=probs)
+      acts.append(act)
+      if act == 0:
+        rews.append(0)
+        break
+      try:
+        lin.apply_opt(actions[act-1])
+        tm = time_linearizer(lin, rawbufs)
+        if math.isinf(tm): raise Exception("failed")
+        rews.append(((last_tm-tm)/base_tm))
+        last_tm = tm
+      except Exception:
+        rews.append(-0.5)
+        break
+      #print(f"{tm*1e6:10.2f}", lin.colored_shape())
+
+    assert len(feats) == len(acts) and len(acts) == len(rews)
+    #print(rews)
+    print(f"***** EPISODE {len(rews)} steps, {sum(rews):5.2f} reward, {base_tm*1e6:12.2f} -> {tm*1e6:12.2f} : {lin.colored_shape()}")
+    all_feats += feats
+    all_acts += acts
+    # rewards to go
+    for i in range(len(rews)-2, -1, -1): rews[i] += rews[i+1]
+    all_rews += rews
+
+    BS = 32
+    if len(all_feats) >= BS:
+      Tensor.training = True
+      x = Tensor(all_feats[:BS])
+      mask = np.zeros((BS, len(actions)+1), dtype=np.float32)
+      mask[range(BS), all_acts[:BS]] = all_rews[:BS]
+      loss = -(net(x) * Tensor(mask)).mean()
+      optim.zero_grad()
+      loss.backward()
+      optim.step()
+      all_feats = all_feats[BS:]
+      all_acts = all_acts[BS:]
+      all_rews = all_rews[BS:]

extra/optimization/run_qnet.py

@@ -0,0 +1,32 @@
+from typing import List, Tuple
+from tinygrad.codegen.opt.kernel import Kernel
+from tinygrad.codegen.opt.search import get_kernel_actions, actions
+
+_net = None
+def beam_q_estimate(beam:List[Tuple[Kernel, float]]) -> List[Tuple[Kernel, float]]:
+  global _net
+  if _net is None:
+    from tinygrad.nn.state import load_state_dict, safe_load
+    from extra.optimization.pretrain_valuenet import ValueNet
+    _net = ValueNet(1021+len(actions), 2)
+    load_state_dict(_net, safe_load("/tmp/qnet.safetensors"), verbose=False)
+  from tinygrad.tensor import Tensor
+  from tinygrad.helpers import Context
+  from extra.optimization.helpers import lin_to_feats
+  import numpy as np
+  feats = []
+  lins = []
+  base_tms = []
+  for lin,tm in beam:
+    lin_feats = lin_to_feats(lin)
+    for a,v in get_kernel_actions(lin, include_0=False).items():
+      acts = np.zeros(len(actions))
+      acts[a-1] = 1.0
+      feats.append(np.concatenate([lin_feats, acts]))
+      lins.append(v)
+      base_tms.append(tm)
+  with Context(BEAM=0):
+    with Tensor.train(False):
+      preds = _net(Tensor(feats)).numpy()
+  pred_time = np.array(base_tms) / np.exp(preds[:, 0])
+  return sorted(zip(lins, pred_time), key=lambda x: x[1])

extra/optimization/search.py

@@ -0,0 +1,34 @@
+import argparse
+from extra.optimization.helpers import ast_str_to_lin, time_linearizer
+
+from tinygrad import dtypes
+from tinygrad.helpers import BEAM, getenv
+from tinygrad.device import Device, Compiled
+from tinygrad.codegen.opt.kernel import Kernel
+from tinygrad.codegen.opt.search import beam_search, bufs_from_lin
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description="Run a search for the optimal opts for a kernel", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument("--ast", type=str, default=None, help="the ast for the kernel to be optimized")
+  parser.add_argument("--file", type=str, default=None, help="a file containing asts to be optimized, one per line")
+  args = parser.parse_args()
+
+  device: Compiled = Device[Device.DEFAULT]
+  print(f"optimizing for {Device.DEFAULT}")
+
+  if args.ast is not None:
+    ast_strs = [args.ast]
+  elif args.file is not None:
+    with open(args.file, 'r') as file:
+      ast_strs = file.readlines()
+
+  for i, ast_str in enumerate(ast_strs):
+    print(f"optimizing {i}/{len(ast_strs)}\nast={ast_str}")
+    lin = ast_str_to_lin(ast_str, opts=device.renderer)
+    rawbufs = bufs_from_lin(lin)
+    lin = beam_search(lin, rawbufs, getenv("BEAM", 8), bool(getenv("BEAM_ESTIMATE", 1)))
+
+    tm = time_linearizer(lin, rawbufs, allow_test_size=False, cnt=10)
+    print(f"final time {tm*1e6:9.0f} us: {lin.colored_shape()}")
+    print(lin.applied_opts)

extra/optimization/test_helpers.py

@@ -0,0 +1,19 @@
+import unittest
+
+from extra.optimization.helpers import load_worlds
+
+class TestKernelDataset(unittest.TestCase):
+  def test_load_worlds_filters(self):
+    all_kernels = load_worlds(filter_reduce=False, filter_noimage=False, filter_novariable=False)
+
+    reduce_kernels = load_worlds(filter_reduce=True, filter_noimage=False, filter_novariable=False)
+    self.assertGreater(len(all_kernels), len(reduce_kernels))
+
+    image_kernels = load_worlds(filter_reduce=False, filter_noimage=True, filter_novariable=False)
+    self.assertGreater(len(all_kernels), len(image_kernels))
+
+    variable_kernels = load_worlds(filter_reduce=False, filter_noimage=False, filter_novariable=True)
+    self.assertGreater(len(all_kernels), len(variable_kernels))
+
+if __name__ == '__main__':
+  unittest.main()

extra/optimization/test_net.py

@@ -0,0 +1,67 @@
+import numpy as np
+import math
+import random
+np.set_printoptions(suppress=True)
+from copy import deepcopy
+from tinygrad.helpers import getenv, colored
+from tinygrad.tensor import Tensor
+from tinygrad.nn.state import get_parameters, get_state_dict, safe_save, safe_load, load_state_dict
+from tinygrad.codegen.opt.search import bufs_from_lin, actions, get_kernel_actions
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from extra.optimization.helpers import load_worlds, ast_str_to_lin, lin_to_feats, time_linearizer
+from extra.optimization.extract_policynet import PolicyNet
+from extra.optimization.pretrain_valuenet import ValueNet
+
+VALUE = getenv("VALUE")
+
+if __name__ == "__main__":
+  if VALUE:
+    net = ValueNet()
+    load_state_dict(net, safe_load("/tmp/valuenet.safetensors"))
+  else:
+    net = PolicyNet()
+    load_state_dict(net, safe_load("/tmp/policynet.safetensors"))
+
+  ast_strs = load_worlds()
+
+  # real randomness
+  random.seed()
+  random.shuffle(ast_strs)
+
+  wins = 0
+  for ep_num,ast_str in enumerate(ast_strs):
+    print("\nEPISODE", ep_num, f"win {wins*100/max(1,ep_num):.2f}%")
+    lin = ast_str_to_lin(ast_str)
+    rawbufs = bufs_from_lin(lin)
+
+    linhc = deepcopy(lin)
+    linhc.applied_opts(hand_coded_optimizations(linhc))
+    tmhc = time_linearizer(linhc, rawbufs)
+    print(f"{tmhc*1e6:10.2f}     HC    ", linhc.colored_shape())
+
+    pred_time = float('nan')
+    tm = float('inf')
+    while 1:
+      if VALUE:
+        acts,feats = [], []
+        for k,v in get_kernel_actions(lin).items():
+          acts.append(k)
+          feats.append(lin_to_feats(v))
+        preds = net(Tensor(feats))
+        pred_time = math.exp(preds.numpy().min())
+        act = acts[preds.numpy().argmin()]
+      else:
+        probs = net(Tensor([lin_to_feats(lin)]))
+        dist = probs.exp().numpy()
+        act = dist.argmax()
+      if act == 0: break
+      try:
+        lin.apply_opt(actions[act-1])
+      except Exception:
+        print("FAILED")
+        break
+      tm = time_linearizer(lin, rawbufs)
+      print(f"{tm*1e6:10.2f} {pred_time*1e6:10.2f}", lin.colored_shape())
+
+    print(f"{colored('BEAT', 'green') if tm < tmhc else colored('lost', 'red')} hand coded {tmhc/tm:5.2f}x")
+    wins += int(tm < tmhc)

extra/optimization/test_time_linearizer.py

@@ -0,0 +1,21 @@
+from extra.optimization.helpers import load_worlds, ast_str_to_lin, time_linearizer
+from tinygrad.codegen.opt.search import bufs_from_lin, get_kernel_actions
+
+if __name__ == "__main__":
+  ast_strs = load_worlds()
+  for i, ast_str in enumerate(ast_strs):
+    lin = ast_str_to_lin(ast_str)
+    rawbufs = bufs_from_lin(lin)
+    test_tm = time_linearizer(lin, rawbufs)
+    if test_tm < 1e-2: continue
+    print(f"EXAMPLE {i}")
+    acted_lins = get_kernel_actions(lin)
+    ok_avg, short_avg = 0, 0
+    for k,v in acted_lins.items():
+      tm1 = time_linearizer(v, rawbufs)
+      tm2 = time_linearizer(v, rawbufs)
+      tm3 = time_linearizer(v, rawbufs, False)
+      print(v.colored_shape(50), f"{tm1*1e3:10.2f} {tm2*1e3:10.2f} {tm3*1e3:10.2f}  :  {((tm1-tm2)/tm1)*100:5.2f}%  vs  {((tm1-tm3)/tm1)*100:5.2f}%")
+      ok_avg += (tm1-tm2)/tm1
+      short_avg += (tm1-tm3)/tm1
+    print(f"{ok_avg/len(acted_lins)*100:5.2f}% vs {short_avg/len(acted_lins)*100:5.2f}%")

extra/thunder/tiny/fa.py

@@ -2,7 +2,7 @@ import math
 
 from tinygrad import Tensor, dtypes
 from tinygrad.helpers import DEBUG
-from tinygrad.uop.ops import UOp, Ops
+from tinygrad.uop.ops import UOp
 
 from extra.thunder.tiny.tk import WARP_THREADS
 from extra.thunder.tiny.tk.kernel import Kernel
@@ -43,18 +43,20 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
   B_local = B // num_devices
   if DEBUG >= 2: print(f"Flash Attention {B=} {B_local=} {N=} {H=} {D=} {H_KV=} {GROUP_SIZE=}")
 
-  def _custom_forward_impl(ou:UOp, l_vecu:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp|None) -> UOp:
+  def custom_forward(ou:UOp, l_vecu:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp) -> UOp:
     with Kernel("fa_custom_forward", (H, N // (Q_BLOCK_SIZE*NUM_WORKERS), B_local), NUM_WORKERS * WARP_THREADS) as ker:
       warp = ker.warp
 
-      o, q, k, v, l_vec = GL(ou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker), GL(l_vecu, ker)
-      mask = GL(masku, ker) if masku is not None else None
+      o, q, k, v, mask, l_vec = GL(ou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker), GL(masku, ker), GL(l_vecu, ker)
 
       head = ker.blockIdx_x
       head_kv = head // GROUP_SIZE
       batch = ker.blockIdx_z
       q_seq = ker.blockIdx_y * NUM_WORKERS + ker.warpid
 
+      k_smem = ker.st((KV_BLOCK_SIZE, D), dtypes.bfloat16)
+      v_smem = ker.st((KV_BLOCK_SIZE, D), dtypes.bfloat16)
+
       q_reg_fl = ker.rt((Q_BLOCK_SIZE, D), dtypes.float32)
       q_reg = ker.rt((Q_BLOCK_SIZE, D), dtypes.bfloat16)
       q_reg_transposed = ker.rt((D, Q_BLOCK_SIZE), dtypes.bfloat16, TileLayout.COL)
@@ -84,10 +86,12 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
       q_reg = warp.copy(q_reg, q_reg_fl)
       q_reg_transposed = warp.transpose(q_reg_transposed, q_reg)
 
-      num_kv_blocks = (q_seq + 1) if is_causal else (N // KV_BLOCK_SIZE)
-      for kv_idx in ker.range(num_kv_blocks):
-        k_reg = warp.load(k_reg, k, (), (batch, kv_idx, head_kv, 0), axis=1)
-        v_reg = warp.load(v_reg, v, (), (batch, kv_idx, head_kv, 0), axis=1)
+      for kv_idx in ker.range(N // KV_BLOCK_SIZE):
+        k_smem = warp.load(k_smem, k, (), (batch, kv_idx, head_kv, 0), axis=1)
+        v_smem = warp.load(v_smem, v, (), (batch, kv_idx, head_kv, 0), axis=1)
+
+        k_reg = warp.load(k_reg, k_smem)
+        v_reg = warp.load(v_reg, v_smem)
 
         # mma qk^t
         att_block = warp.zero(att_block.after(kv_idx))
@@ -95,16 +99,9 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
         att_block = warp.mma_AtB(att_block, k_reg_transposed, q_reg_transposed)
 
         # apply attention mask
-        if is_causal:
-          bs_rows, bs_cols, bs_stride = att_block.base_shape.rows, att_block.base_shape.cols, att_block.base_shape.stride
-          q_base = q_seq * Q_BLOCK_SIZE + (warp.laneid % bs_cols)
-          kv_base = kv_idx * KV_BLOCK_SIZE + (warp.laneid // bs_cols) * bs_stride
-          att_block = warp.map(att_block,
-            lambda x, idx: ((kv_base + idx[0]*bs_rows + idx[2]) > (q_base + idx[1]*bs_cols)).alu(Ops.WHERE, UOp.ufix(x._uop, -math.inf), x))
-        elif mask is not None:
-          mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_seq, kv_idx), axis=2)
-          mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
-          att_block += mask_reg_transposed
+        mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_seq, kv_idx), axis=2)
+        mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
+        att_block += mask_reg_transposed
 
         # softmax
         max_vec_last = warp.copy(max_vec_last.after(kv_idx), max_vec)
@@ -144,18 +141,11 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
 
       return ker.finish()
 
-  def custom_forward_causal(ou:UOp, l_vecu:UOp, qu:UOp, ku:UOp, vu:UOp) -> UOp:
-    return _custom_forward_impl(ou, l_vecu, qu, ku, vu, None)
-
-  def custom_forward_masked(ou:UOp, l_vecu:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp) -> UOp:
-    return _custom_forward_impl(ou, l_vecu, qu, ku, vu, masku)
-
-  def _custom_backward_q_impl(dqu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp|None, l_vecu:UOp, delta_vecu:UOp) -> UOp:
+  def custom_backward_q(dqu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp, l_vecu:UOp, delta_vecu:UOp) -> UOp:
     with Kernel("fa_custom_backward_q", (H, N // (Q_BLOCK_SIZE*NUM_WORKERS), B_local), NUM_WORKERS * WARP_THREADS) as ker:
       warp = ker.warp
 
-      dq, do, q, k, v = GL(dqu, ker), GL(dou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker)
-      mask = GL(masku, ker) if masku is not None else None
+      dq, do, q, k, v, mask = GL(dqu, ker), GL(dou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker), GL(masku, ker)
       l_vec, delta_vec = GL(l_vecu, ker), GL(delta_vecu, ker)
 
       head = ker.blockIdx_x
@@ -163,6 +153,9 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
       batch = ker.blockIdx_z
       q_seq = ker.blockIdx_y * NUM_WORKERS + ker.warpid
 
+      k_smem = ker.st((KV_BLOCK_SIZE, D), dtypes.bfloat16)
+      v_smem = ker.st((KV_BLOCK_SIZE, D), dtypes.bfloat16)
+
       q_reg_fl = ker.rt((Q_BLOCK_SIZE, D), dtypes.float32)
       q_reg = ker.rt((Q_BLOCK_SIZE, D), dtypes.bfloat16)
       q_reg_t = ker.rt((D, Q_BLOCK_SIZE), dtypes.bfloat16, TileLayout.COL)
@@ -201,30 +194,24 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
       l_vec_reg *= 1.0 / math.log(2)
       delta_vec_reg = warp.load(delta_vec_reg, delta_vec, (), (batch, head, 0, q_seq), axis=2)
 
-      num_kv_blocks = (q_seq + 1) if is_causal else (N // KV_BLOCK_SIZE)
-      for kv_idx in ker.range(num_kv_blocks):
-        k_reg = warp.load(k_reg, k, (), (batch, kv_idx, head_kv, 0), axis=1)
-        k_reg_col = warp.load(k_reg_col, k, (), (batch, kv_idx, head_kv, 0), axis=1)
-        v_reg = warp.load(v_reg, v, (), (batch, kv_idx, head_kv, 0), axis=1)
+      for kv_idx in ker.range(N // KV_BLOCK_SIZE):
+        k_smem = warp.load(k_smem, k, (), (batch, kv_idx, head_kv, 0), axis=1)
+        v_smem = warp.load(v_smem, v, (), (batch, kv_idx, head_kv, 0), axis=1)
 
+        k_reg = warp.load(k_reg, k_smem)
         k_reg_t = warp.transpose(k_reg_t, k_reg)
+        k_reg_col = warp.load(k_reg_col, k_smem)
         k_reg_col_t = warp.transpose(k_reg_col_t, k_reg_col)
+        v_reg = warp.load(v_reg, v_smem)
 
         # mma qk^t
         att_block = warp.zero(att_block.after(kv_idx))
         att_block = warp.mma_AtB(att_block, k_reg_t, q_reg_t)
 
         # apply attention mask
-        if is_causal:
-          bs_rows, bs_cols, bs_stride = att_block.base_shape.rows, att_block.base_shape.cols, att_block.base_shape.stride
-          q_base = q_seq * Q_BLOCK_SIZE + (warp.laneid % bs_cols)
-          kv_base = kv_idx * KV_BLOCK_SIZE + (warp.laneid // bs_cols) * bs_stride
-          att_block = warp.map(att_block,
-            lambda x, idx: ((kv_base + idx[0]*bs_rows + idx[2]) > (q_base + idx[1]*bs_cols)).alu(Ops.WHERE, UOp.ufix(x._uop, -math.inf), x))
-        elif mask is not None:
-          mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_seq, kv_idx), axis=2)
-          mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
-          att_block += mask_reg_transposed
+        mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_seq, kv_idx), axis=2)
+        mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
+        att_block += mask_reg_transposed
 
         att_block -= l_vec_reg
         att_block = att_block.exp2()
@@ -244,24 +231,19 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
 
       return ker.finish()
 
-  def custom_backward_q_causal(dqu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, l_vecu:UOp, delta_vecu:UOp) -> UOp:
-    return _custom_backward_q_impl(dqu, dou, qu, ku, vu, None, l_vecu, delta_vecu)
-
-  def custom_backward_q_masked(dqu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp, l_vecu:UOp, delta_vecu:UOp) -> UOp:
-    return _custom_backward_q_impl(dqu, dou, qu, ku, vu, masku, l_vecu, delta_vecu)
-
-  def _custom_backward_kv_impl(dku:UOp, dvu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp|None, l_vecu:UOp, delta_vecu:UOp):
+  def custom_backward_kv(dku:UOp, dvu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp, l_vecu:UOp, delta_vecu:UOp):
     with Kernel("fa_custom_backward_kv", (H_KV, N // (KV_BLOCK_SIZE*NUM_WORKERS), B_local), NUM_WORKERS * WARP_THREADS) as ker:
       warp = ker.warp
 
-      dk, dv, do, q, k, v = GL(dku, ker), GL(dvu, ker), GL(dou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker)
-      mask = GL(masku, ker) if masku is not None else None
+      dk, dv, do, q, k, v, mask = GL(dku, ker), GL(dvu, ker), GL(dou, ker), GL(qu, ker), GL(ku, ker), GL(vu, ker), GL(masku, ker)
       l_vec, delta_vec = GL(l_vecu, ker), GL(delta_vecu, ker)
 
       head_kv = ker.blockIdx_x
       batch = ker.blockIdx_z
       kv_seq = ker.blockIdx_y * NUM_WORKERS + ker.warpid
 
+      q_smem = ker.st((Q_BLOCK_SIZE, D), dtypes.bfloat16)
+      do_smem = ker.st((Q_BLOCK_SIZE, D), dtypes.bfloat16)
       att_smem = ker.st((Q_BLOCK_SIZE, KV_BLOCK_SIZE), dtypes.bfloat16)
 
       q_reg = ker.rt((Q_BLOCK_SIZE, D), dtypes.bfloat16)
@@ -295,17 +277,19 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
       k_reg_t = warp.transpose(k_reg_t, k_reg)
       v_reg = warp.load(v_reg, v, (), (batch, kv_seq, head_kv, 0), axis=1)
 
-      q_start = kv_seq if is_causal else 0
-      for q_idx in ker.range(q_start, N // Q_BLOCK_SIZE):
+      for q_idx in ker.range(N // Q_BLOCK_SIZE):
         for g in ker.range(GROUP_SIZE):
           head_q = head_kv * GROUP_SIZE + g
 
-          q_reg = warp.load(q_reg, q, (), (batch, q_idx, head_q, 0), axis=1)
-          q_reg_col = warp.load(q_reg_col, q, (), (batch, q_idx, head_q, 0), axis=1)
-          do_reg = warp.load(do_reg, do, (), (batch, q_idx, head_q, 0), axis=1)
-          do_reg_col = warp.load(do_reg_col, do, (), (batch, q_idx, head_q, 0), axis=1)
+          # load q and do
+          q_smem = warp.load(q_smem, q, (), (batch, q_idx, head_q, 0), axis=1)
+          do_smem = warp.load(do_smem, do, (), (batch, q_idx, head_q, 0), axis=1)
 
+          q_reg = warp.load(q_reg, q_smem)
           q_reg_t = warp.transpose(q_reg_t, q_reg)
+          q_reg_col = warp.load(q_reg_col, q_smem)
+          do_reg = warp.load(do_reg, do_smem)
+          do_reg_col = warp.load(do_reg_col, do_smem)
 
           # load l_vec and delta_vec
           l_vec_reg = warp.load(l_vec_reg, l_vec, (), (batch, head_q, 0, q_idx), axis=2)
@@ -318,16 +302,9 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
           att_block *= (1.0 / math.sqrt(D)) * (1.0 / math.log(2))
 
           # apply attention mask
-          if is_causal:
-            bs_rows, bs_cols, bs_stride = att_block.base_shape.rows, att_block.base_shape.cols, att_block.base_shape.stride
-            q_base = q_idx * Q_BLOCK_SIZE + (warp.laneid % bs_cols)
-            kv_base = kv_seq * KV_BLOCK_SIZE + (warp.laneid // bs_cols) * bs_stride
-            att_block = warp.map(att_block,
-              lambda x, idx: ((kv_base + idx[0]*bs_rows + idx[2]) > (q_base + idx[1]*bs_cols)).alu(Ops.WHERE, UOp.ufix(x._uop, -math.inf), x))
-          elif mask is not None:
-            mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_idx, kv_seq), axis=2)
-            mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
-            att_block += mask_reg_transposed
+          mask_reg = warp.load(mask_reg, mask, (), (batch, 0, q_idx, kv_seq), axis=2)
+          mask_reg_transposed = warp.transpose(mask_reg_transposed, mask_reg)
+          att_block += mask_reg_transposed
 
           att_block -= l_vec_reg
           att_block = att_block.exp2()
@@ -359,31 +336,24 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
 
       return ker.finish(2)
 
-  def custom_backward_kv_causal(dku:UOp, dvu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, l_vecu:UOp, delta_vecu:UOp):
-    return _custom_backward_kv_impl(dku, dvu, dou, qu, ku, vu, None, l_vecu, delta_vecu)
-
-  def custom_backward_kv_masked(dku:UOp, dvu:UOp, dou:UOp, qu:UOp, ku:UOp, vu:UOp, masku:UOp, l_vecu:UOp, delta_vecu:UOp):
-    return _custom_backward_kv_impl(dku, dvu, dou, qu, ku, vu, masku, l_vecu, delta_vecu)
-
   single_device = xq.device[0] if isinstance(xq.device, tuple) else xq.device
 
   if is_causal:
     if attn_mask is not None: raise RuntimeError("cannot set attn_mask when is_causal=True")
-  elif attn_mask is not None:
+    attn_mask = Tensor.ones((B, 1, N, N), requires_grad=False, device=single_device, dtype=dtypes.bool).tril()
+  if attn_mask is not None:
     if attn_mask.dtype == dtypes.bool: attn_mask = attn_mask.where(0, -float("inf"))
-    if attn_mask.shape != (B, 1, N, N):
-      attn_mask = attn_mask.expand(B, 1, N, N)
-    if isinstance(xq.device, tuple) and not isinstance(attn_mask.device, tuple):
-      attn_mask = attn_mask.shard(xq.device, axis=0)
   else:
     attn_mask = Tensor.zeros((B, 1, N, N), requires_grad=False, device=single_device, dtype=dtypes.float32)
-    if isinstance(xq.device, tuple):
-      attn_mask = attn_mask.shard(xq.device, axis=0)
+  if attn_mask.shape != (B, 1, N, N):
+    attn_mask = attn_mask.expand(B, 1, N, N)
+  if isinstance(xq.device, tuple) and not isinstance(attn_mask.device, tuple):
+    attn_mask = attn_mask.shard(xq.device, axis=0)
 
   attn = _sharded_empty_like(xq, axis=0)
   l_vec = _sharded_empty((B, H, 1, N), xq, axis=0)
 
-  def grad_causal(gradu:UOp, _) -> tuple[None, None, UOp, UOp, UOp]:
+  def grad(gradu:UOp, _) -> tuple[None, None, UOp, UOp, UOp, None]:
     grad = Tensor(gradu, device=gradu.device)
     grad_q = _sharded_empty_like(xq, axis=0)
     grad_k = _sharded_empty_like(xk, axis=0)
@@ -391,26 +361,11 @@ def flash_attention(xq, xk, xv, attn_mask:Tensor|None=None, is_causal:bool=False
 
     delta_vec = (grad * attn).sum(-1, dtype=dtypes.float32).transpose(1, 2).unsqueeze(-2).detach()
 
-    grad_q = Tensor.custom_kernel(grad_q, grad, xq, xk, xv, l_vec, delta_vec, fxn=custom_backward_q_causal)[0]
-    grad_k, grad_v = Tensor.custom_kernel(grad_k, grad_v, grad, xq, xk, xv, l_vec, delta_vec, fxn=custom_backward_kv_causal)[:2]
-    return (None, None, grad_q.uop, grad_k.uop, grad_v.uop)
-
-  def grad_masked(gradu:UOp, _) -> tuple[None, None, UOp, UOp, UOp, None]:
-    grad = Tensor(gradu, device=gradu.device)
-    grad_q = _sharded_empty_like(xq, axis=0)
-    grad_k = _sharded_empty_like(xk, axis=0)
-    grad_v = _sharded_empty_like(xv, axis=0)
-
-    delta_vec = (grad * attn).sum(-1, dtype=dtypes.float32).transpose(1, 2).unsqueeze(-2).detach()
-
-    grad_q = Tensor.custom_kernel(grad_q, grad, xq, xk, xv, attn_mask, l_vec, delta_vec, fxn=custom_backward_q_masked)[0]
-    grad_k, grad_v = Tensor.custom_kernel(grad_k, grad_v, grad, xq, xk, xv, attn_mask, l_vec, delta_vec, fxn=custom_backward_kv_masked)[:2]
+    grad_q = Tensor.custom_kernel(grad_q, grad, xq, xk, xv, attn_mask, l_vec, delta_vec, fxn=custom_backward_q)[0]
+    grad_k, grad_v = Tensor.custom_kernel(grad_k, grad_v, grad, xq, xk, xv, attn_mask, l_vec, delta_vec, fxn=custom_backward_kv)[:2]
     return (None, None, grad_q.uop, grad_k.uop, grad_v.uop, None)
 
-  if is_causal:
-    attn, l_vec = Tensor.custom_kernel(attn, l_vec, xq, xk, xv, fxn=custom_forward_causal, grad_fxn=grad_causal)[:2]
-  else:
-    attn, l_vec = Tensor.custom_kernel(attn, l_vec, xq, xk, xv, attn_mask, fxn=custom_forward_masked, grad_fxn=grad_masked)[:2]
+  attn, l_vec = Tensor.custom_kernel(attn, l_vec, xq, xk, xv, attn_mask, fxn=custom_forward, grad_fxn=grad)[:2]
   attn_ = attn[:, :N_, :, :D_]
 
   return attn_.transpose(1, 2).cast(odtype)

extra/thunder/tiny/tk/kernel.py

@@ -12,7 +12,7 @@ class _tk_range:
   def __next__(self):
     if not self.done:
       self.done = True
-      self._rng = UOp.range((self.end - self.start) // self.step, self.rid, axis_type=self.axis_type) * self.step + self.start
+      self._rng = UOp.range(self.end // self.step, self.rid, axis_type=self.axis_type) * self.step + self.start
       return self._rng
     raise StopIteration
 

extra/torch_backend/backend.py

@@ -29,10 +29,6 @@ def wrap(x: Tensor) -> torch.Tensor:
   x._strides = strides_for_shape(x.shape) # always recalculate
   if (not hasattr(x, '_storage_offset')) or (not x.uop.is_realized): x._storage_offset = calculate_storage_offset(x)
   return mod.wrap(x, _to_torch_dtype(x.dtype), _to_torch_device(x.device).index)
-def _update_torch_metadata(tensor: torch.Tensor, tiny: Tensor) -> None:
-  tiny._strides = strides_for_shape(tiny.shape)
-  tiny._storage_offset = calculate_storage_offset(tiny)
-  mod.update_metadata(tensor, tiny.shape, tiny._strides, tiny._storage_offset)
 def unwrap(x:torch.Tensor) -> Tensor:
   assert isinstance(x, torch.Tensor), f"x isn't {type(x)}"
   return mod.unwrap(x)
@@ -348,7 +344,7 @@ def scatter_add(self, dim, index, src, out):
 def _copy_between_devices(src, dest, cast_dtype, to_device, non_blocking=False):
   if src.is_tiny and dest.is_tiny:
     src_t, dest_t = unwrap(src), unwrap(dest)
-    if dest_t.uop.has_buffer_identity() or dest_t.uop.is_realized: src_t = src_t.contiguous()
+    if dest_t.uop.is_contiguous() or dest_t.uop.is_realized: src_t = src_t.contiguous()
     _apply_inplace(dest_t, src_t.cast(cast_dtype).to(to_device))
   elif src.is_tiny and dest.is_cpu:
     dest.resize_(src.numel()).resize_(src.shape)
@@ -615,10 +611,7 @@ tiny_backend = {**{k:wrap_out(v) for k,v in tiny_backend_out.items()}, **{
   "aten.fill_.Tensor": lambda self, value: Tensor.full(self.shape, value.reshape(()).item(), device=self.device, dtype=self.dtype),
   "aten.flip": Tensor.flip,
   "aten.scatter_reduce.two": Tensor.scatter_reduce,
-  "aten.squeeze_.dim": Tensor.squeeze,
-  "aten.unsqueeze_": Tensor.unsqueeze,
-  "aten.transpose_": Tensor.transpose,
-  "aten.t_": Tensor.transpose,
+  "aten.squeeze_.dim": lambda self, dim: self.replace(self.squeeze(dim), allow_shape_mismatch=True), # TODO: inplace view op, here?
   "aten.add.Tensor": lambda input,other,alpha=1: input+alpha*other,
   "aten.linspace": lambda start, stop, steps, dtype=None, **kwargs:
     Tensor.linspace(start, stop, steps, **({"dtype": _from_torch_dtype(dtype)} if dtype is not None else {})),
@@ -662,13 +655,6 @@ inplace_ops = {
   "aten.masked_fill_.Tensor",
 }
 
-inplace_view_ops = {
-  "aten.squeeze_.dim",
-  "aten.unsqueeze_",
-  "aten.transpose_",
-  "aten.t_",
-}
-
 def wrap_fxn(k,f):
   def nf(*args, **kwargs):
     if TORCH_DEBUG:
@@ -689,42 +675,8 @@ def wrap_inplace(k,f):
     return orig
   return nf
 
-def wrap_inplace_view_op(k,f):
-  def nf(*args, **kwargs):
-    orig = args[0]
-    args, kwargs = unwrap_args(args, kwargs)
-    target = args[0]
-    new_view = f(*args, **kwargs)
-    if new_view is target or new_view.uop is target.uop:
-      _update_torch_metadata(orig, target)
-      return orig
-    base = canonical_base(target)
-    op = (f, args[1:], kwargs)
-    if target is base:
-      views = derived_views(base)
-      if views:
-        old_base = Tensor(base.uop, device=base.device)
-        old_base.requires_grad = base.requires_grad
-        old_base._views = getattr(base, "_views", set())
-        for v in views: v._view_base = old_base
-        base._views = set()
-        base._view_base = old_base
-        base._view_ops = [op]
-        old_base._views.add(weakref.ref(base))
-    else:
-      target._view_base = base
-      base._views = getattr(base, "_views", set())
-      base._views.add(weakref.ref(target))
-      target._view_ops = _get_view_ops(target) + [op]
-    target.uop = new_view.uop
-    _update_torch_metadata(orig, target)
-    return orig
-  return nf
-
 for k,v in tiny_backend.items():
-  if k in inplace_view_ops: wrapper = wrap_inplace_view_op
-  elif k in inplace_ops: wrapper = wrap_inplace
-  else: wrapper = wrap_fxn
+  wrapper = wrap_inplace if k in inplace_ops else wrap_fxn
   torch.library.impl(k.replace("aten.", "aten::"), "privateuseone")(wrapper(k,v))
 
 @torch.library.impl("aten::equal", "privateuseone")

extra/torch_backend/test_inplace.py

@@ -67,37 +67,5 @@ class TestTorchBackendInplace(unittest.TestCase):
     d += torch.arange(4)
     np.testing.assert_array_equal(a.cpu(), torch.arange(4).cpu())
 
-  def test_inplace_view_metadata(self):
-    a = torch.arange(6, dtype=torch.float32).reshape(1, 2, 3)
-    ret = a.squeeze_(0)
-    self.assertIs(ret, a)
-    self.assertEqual(a.shape, torch.Size([2, 3]))
-    ret = a.unsqueeze_(1)
-    self.assertIs(ret, a)
-    self.assertEqual(a.shape, torch.Size([2, 1, 3]))
-    ret = a.transpose_(0, 2)
-    self.assertIs(ret, a)
-    self.assertEqual(a.shape, torch.Size([3, 1, 2]))
-
-  def test_t_inplace_metadata(self):
-    a = torch.arange(6, dtype=torch.float32).reshape(2, 3)
-    ret = a.t_()
-    self.assertIs(ret, a)
-    self.assertEqual(a.shape, torch.Size([3, 2]))
-    expected = torch.arange(6, dtype=torch.float32).reshape(2, 3).t()
-    np.testing.assert_array_equal(a.cpu().numpy(), expected.cpu().numpy())
-
-  def test_squeeze_matmul(self):
-    # squeeze_ is used internally by PyTorch for vector-matrix matmul (unsqueeze -> mm -> squeeze_)
-    a = torch.arange(65, dtype=torch.float32)
-    b = torch.arange(65*45, dtype=torch.float32).reshape(65, 45)
-    result = a.matmul(b)
-    self.assertEqual(result.shape, torch.Size([45]))
-    # verify correctness
-    a_cpu = torch.arange(65, dtype=torch.float32, device='cpu')
-    b_cpu = torch.arange(65*45, dtype=torch.float32, device='cpu').reshape(65, 45)
-    expected = a_cpu.matmul(b_cpu)
-    np.testing.assert_allclose(result.cpu().numpy(), expected.numpy(), rtol=1e-4, atol=1e-4)
-
 if __name__ == "__main__":
   unittest.main()

extra/torch_backend/wrapped_tensor.cpp

@@ -131,15 +131,7 @@ py::object unwrap_tensor(const at::Tensor &tensor) {
   return py::reinterpret_borrow<py::object>(tiny->ptr(getPyInterpreter()));
 }
 
-void update_metadata(const at::Tensor &tensor, const std::vector<int64_t> &sizes,
-                     const std::vector<int64_t> &strides, int64_t storage_offset) {
-  auto* impl = tensor.unsafeGetTensorImpl();
-  impl->set_allow_tensor_metadata_change(true);
-  impl->set_sizes_and_strides(sizes, strides, storage_offset);
-}
-
 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
   m.def("wrap", &wrap_tensor);
   m.def("unwrap", &unwrap_tensor);
-  m.def("update_metadata", &update_metadata);
 }

extra/usbgpu/patch.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-import os, zlib, struct, hashlib
-from tinygrad.helpers import getenv
+import sys, os, zlib, struct, hashlib
+from tinygrad.helpers import DEBUG, getenv, fetch
 from tinygrad.runtime.support.usb import USB3
 
 SUPPORTED_CONTROLLERS = [
@@ -50,7 +50,7 @@ patched_fw = patch(file_path, file_hash, patches)
 dev = None
 for vendor, device in SUPPORTED_CONTROLLERS:
   try:
-    dev = USB3(vendor, device, 0x81, 0x83, 0x02, 0x04, use_bot=True)
+    dev = USB3(vendor, device, 0x81, 0x83, 0x02, 0x04)
     break
   except RuntimeError: pass
 if dev is None:

extra/viz/cli.py

@@ -3,8 +3,7 @@ import argparse, pathlib
 from typing import Iterator
 from tinygrad.viz import serve as viz
 from tinygrad.uop.ops import RewriteTrace
-from tinygrad.helpers import temp, ansistrip, colored, time_to_str, ansilen
-from test.null.test_viz import load_profile
+from tinygrad.helpers import temp, ansistrip, colored
 
 def optional_eq(val:dict, arg:str|None) -> bool: return arg is None or ansistrip(val["name"]) == arg
 
@@ -28,46 +27,11 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser()
   parser.add_argument('--kernel', type=str, default=None, metavar="NAME", help='Select a kernel by name (optional name, default: only list names)')
   parser.add_argument('--select', type=str, default=None, metavar="NAME",
-                      help='Rewrites: Select an item within the chosen kernel (optional name, default: only list names)')
-  parser.add_argument('--profile', action="store_true", help="View profiling trace (default: views rewrites)")
-  parser.add_argument('--device', type=str, default=None, metavar="NAME", help="Profile only: Select a device (default: prints all devices)")
-  parser.add_argument('--profile-path', type=pathlib.Path, metavar="PATH", help='Path to profile (optional file, default: latest profile)',
-                      default=pathlib.Path(temp("profile.pkl", append_user=True)))
-  parser.add_argument('--rewrites-path', type=pathlib.Path, metavar="PATH", help='Path to rewrites (optional file, default: latest rewrites)',
-                      default=pathlib.Path(temp("rewrites.pkl", append_user=True)))
+                      help='Select an item within the chosen kernel (optional name, default: only list names)')
   args = parser.parse_args()
 
-  viz.trace = viz.load_pickle(args.rewrites_path, default=RewriteTrace([], [], {}))
+  viz.trace = viz.load_pickle(pathlib.Path(temp("rewrites.pkl", append_user=True)), default=RewriteTrace([], [], {}))
   viz.ctxs = viz.get_rewrites(viz.trace)
-
-  if args.profile:
-    from tabulate import tabulate
-    profile = load_profile(viz.load_pickle(args.profile_path, default=[]))
-    agg, total, n = {}, 0, 0
-    for k,v in profile["layout"].items():
-      if not optional_eq({"name":k}, args.device): continue
-      print(k)
-      if args.device is None: continue
-      for e in v.get("events", []):
-        et = e["dur"]*1e-6
-        if args.kernel is not None:
-          if ansistrip(e["name"]) == args.kernel and n < 10:
-            ptm = colored(time_to_str(et, w=9), "yellow" if et > 0.01 else None) if et is not None else ""
-            name = e["name"]+(" " * (46 - ansilen(e["name"])))
-            print(f"{name} {ptm}/{(et or 0)*1e3:9.2f}ms  "+e['fmt'].replace('\n', ' | ')+"  ")
-            n += 1
-        else:
-          a = agg.setdefault(e["name"], [0.0, 0])
-          a[0] += et
-          a[1] += 1
-          total += et
-
-    if agg:
-      rows = [[n, t, time_to_str(t, w=9), t / c if c else 0.0, c, (t / total * 100.0) if total else 0.0] for n, (t, c) in agg.items()]
-      rows.sort(key=lambda r: r[1], reverse=True)
-      print(tabulate([[r[0], r[2], r[4], f"{r[5]:.2f}%"] for r in rows[:30]], headers=["name", "total", "count", "pct"], tablefmt="github"))
-    exit(0)
-
   for k in viz.ctxs:
     if not optional_eq(k, args.kernel): continue
     print(k["name"])

test/device/test_hcq.py

@@ -119,7 +119,7 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
 
   def test_exec_2_kernels_100_times(self):
@@ -135,7 +135,7 @@ class TestHCQ(unittest.TestCase):
       q.submit(TestHCQ.d0, {virt_val.expr: TestHCQ.d0.timeline_value})
       TestHCQ.d0.timeline_value += 1
 
-    val = TestHCQ.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 200.0, f"got val {val}"
 
   @unittest.skipIf(Device.DEFAULT in {"CPU"}, "No globals/locals on LLVM/CPU")
@@ -151,9 +151,9 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 0.0, f"got val {val}, should not be updated"
 
   @unittest.skipIf(Device.DEFAULT in {"CPU"}, "No globals/locals on LLVM/CPU")
@@ -186,7 +186,7 @@ class TestHCQ(unittest.TestCase):
           TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
           TestHCQ.d0.timeline_value += 1
 
-          res_sum = sum(x for x in zt.as_memoryview().cast("I"))
+          res_sum = sum(x for x in zt.as_buffer().cast("I"))
           assert x * y * z == res_sum, f"want {x * y * z}, got {res_sum}"
 
   # Test copy
@@ -200,7 +200,7 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 1.0, f"got val {val}"
 
   def test_copy_long(self):
@@ -218,7 +218,7 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    mv_buf1 = buf1.as_memoryview().cast('Q')
+    mv_buf1 = buf1.as_buffer().cast('Q')
     assert libc.memcmp(mv_address(mv_buf1), buf2._buf.va_addr, sz) == 0
 
   @slow
@@ -242,7 +242,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      mv_buf1 = buf1.as_memoryview()
+      mv_buf1 = buf1.as_buffer()
       assert libc.memcmp(mv_address(mv_buf1), buf2._buf.va_addr, sz) == 0
 
   def test_update_copy(self):
@@ -260,7 +260,7 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 1.0, f"got val {val}"
 
   def test_update_copy_long(self):
@@ -283,7 +283,7 @@ class TestHCQ(unittest.TestCase):
     TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
 
-    mv_buf1 = buf1.as_memoryview().cast('Q')
+    mv_buf1 = buf1.as_buffer().cast('Q')
     for i in range(sz//8): assert mv_buf1[i] == 0x0101010101010101, f"offset {i*8} differs, not all copied, got {hex(mv_buf1[i])}"
 
   # Test bind api
@@ -421,7 +421,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      assert buf1.as_memoryview()[0] == i
+      assert buf1.as_buffer()[0] == i
 
   def test_small_copies_from_host_buf_intercopy(self):
     if TestHCQ.d0.hw_copy_queue_t is None: self.skipTest("device does not support copy queue")
@@ -440,7 +440,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      assert buf2.as_memoryview()[0] == i
+      assert buf2.as_buffer()[0] == i
 
   def test_small_copies_from_host_buf_transfer(self):
     if TestHCQ.d0.hw_copy_queue_t is None: self.skipTest("device does not support copy queue")
@@ -463,7 +463,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      assert buf2.as_memoryview()[0] == i
+      assert buf2.as_buffer()[0] == i
 
   def test_memory_barrier(self):
     a = Tensor([0, 1], device=Device.DEFAULT, dtype=dtypes.int8).realize()
@@ -486,7 +486,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      assert buf1.as_memoryview()[0] == (i + 1), f"has {buf1.as_memoryview()[0]}, need {i + 1}"
+      assert buf1.as_buffer()[0] == (i + 1), f"has {buf1.as_buffer()[0]}, need {i + 1}"
 
   def test_memory_barrier_before_copy(self):
     if TestHCQ.d0.hw_copy_queue_t is None: self.skipTest("device does not support copy queue")
@@ -511,7 +511,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.d0.timeline_signal.wait(TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
 
-      assert buf2.as_memoryview()[0] == i
+      assert buf2.as_buffer()[0] == i
 
   def test_map_cpu_buffer_to_device(self):
     if Device[Device.DEFAULT].hw_copy_queue_t is None: self.skipTest("skip device without copy queue")

test/external/external_benchmark_hcopt.py

@@ -0,0 +1,43 @@
+import random
+from tinygrad.helpers import getenv
+from tinygrad.codegen.opt.search import beam_search, bufs_from_lin
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from extra.optimization.helpers import load_worlds, ast_str_to_lin, time_linearizer
+
+def optimize_kernel(k):
+  # TODO: update this
+  return hand_coded_optimizations(k)
+
+if __name__ == '__main__':
+  hcopt_wins = beam_wins = tie = 0
+  hcopt_total = beam_total = 0.0
+
+  worlds = load_worlds(filter_reduce=False, filter_noimage=True, filter_novariable=False)
+  random.seed(0)
+  random.shuffle(worlds)
+
+  for world in worlds[:500]:
+    k = ast_str_to_lin(world)
+    rawbufs = bufs_from_lin(k)
+
+    k_hcopt = k.copy()
+    k_hcopt.apply_opts(optimize_kernel(k_hcopt))
+    k_beam = beam_search(k.copy(), rawbufs, getenv("BEAM", 2))
+
+    disable_cache = bool(getenv("NOCACHE", 0))
+    t_hcopt = time_linearizer(k_hcopt, rawbufs, allow_test_size=False, cnt=10, disable_cache=disable_cache, clear_l2=True) * 1e6
+    t_beam = time_linearizer(k_beam, rawbufs, allow_test_size=False, cnt=10, disable_cache=disable_cache, clear_l2=True) * 1e6
+
+    if t_hcopt == t_beam: tie += 1
+    elif t_hcopt < t_beam: hcopt_wins += 1
+    else: beam_wins += 1
+    hcopt_total += t_hcopt
+    beam_total += t_beam
+
+    print(f"{t_hcopt=:5.2f} {k_hcopt.applied_opts=}")
+    print("")
+    print(f"{t_beam=:5.2f} {k_beam.applied_opts=}")
+    print("*"*20)
+
+  print(f"{hcopt_wins=}, {beam_wins=}, {tie=}")
+  print(f"{hcopt_total=:.2f}, {beam_total=:.2f}")

test/external/external_benchmark_op_conv.py

@@ -11,7 +11,7 @@ from tinygrad.dtype import ImageDType, Invalid
 # PYTHONPATH="." DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_vision.onnx
 
 def vision_conv_143():
-  c0 = UOp(Ops.PARAM, dtypes.imageh((16, 1024, 4)), (), 0)
+  c0 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((16, 1024, 4)), (), 0)
   c2 = UOp.range(32, 3, AxisType.LOOP)
   c5 = UOp.range(128, 4, AxisType.LOOP)
   c8 = UOp.range(16, 2, AxisType.LOOP)
@@ -21,13 +21,13 @@ def vision_conv_143():
   c26 = UOp.range(7, 1, AxisType.REDUCE)
   c27 = c2*2+c26
   c32 = ((c27<3)!=True)&(c27<67)
-  c34 = UOp(Ops.PARAM, dtypes.imageh((32, 1024, 4)), (), 1)
+  c34 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((32, 1024, 4)), (), 1)
   c38 = c5//2
   c45 = (c32&c24).where((c27*64+c38+c17*4096+-12480), UOp.const(dtypes.index, Invalid))
   c48 = (c24&c32).where(c34.index(c45), UOp.const(dtypes.float, 0.0))
-  c49 = UOp(Ops.PARAM, dtypes.imageh((64, 49, 4)), (), 2)
+  c49 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((64, 49, 4)), (), 2)
   c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196))
-  c63 = UOp(Ops.PARAM, dtypes.float.ptr(128), (), 3)
+  c63 = UOp(Ops.DEFINE_GLOBAL, dtypes.float.ptr(128), (), 3)
   c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5)
   c67 = c0.index((c2*128+c5+c8*4096), ptr=True).store(c65).end(c8, c2, c5)
 
@@ -37,7 +37,7 @@ def vision_conv_143():
   return c67.sink(arg=KernelInfo(name="conv", opts_to_apply=opts))
 
 def vision_conv_153():
-  c0 = UOp(Ops.PARAM, dtypes.imageh((8, 1024, 4)), (), 0)
+  c0 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((8, 1024, 4)), (), 0)
   c2 = UOp.range(16, 3, AxisType.LOOP)
   c5 = UOp.range(256, 4, AxisType.LOOP)
   c8 = UOp.range(8, 2, AxisType.LOOP)
@@ -47,13 +47,13 @@ def vision_conv_153():
   c26 = UOp.range(7, 1, AxisType.REDUCE)
   c27 = c2*2+c26
   c32 = ((c27<3)!=True)&(c27<35)
-  c34 = UOp(Ops.PARAM, dtypes.imageh((16, 1024, 4)), (), 1)
+  c34 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((16, 1024, 4)), (), 1)
   c38 = c5//2
   c45 = (c32&c24).where((c27*128+c38+c17*4096+-12672), UOp.const(dtypes.index, Invalid))
   c48 = (c24&c32).where(c34.index(c45), UOp.const(dtypes.float, 0.0))
-  c49 = UOp(Ops.PARAM, dtypes.imageh((128, 49, 4)), (), 2)
+  c49 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((128, 49, 4)), (), 2)
   c61 = c48*c49.index((c26*4+c5%2+c16*28+c38*196))
-  c63 = UOp(Ops.PARAM, dtypes.float.ptr(256), (), 3)
+  c63 = UOp(Ops.DEFINE_GLOBAL, dtypes.float.ptr(256), (), 3)
   c65 = c61.reduce(c16, c26, arg=Ops.ADD)+c63.index(c5)
   c67 = c0.index((c2*256+c5+c8*4096), ptr=True).store(c65).end(c8, c2, c5)
 
@@ -63,16 +63,16 @@ def vision_conv_153():
   return c67.sink(arg=KernelInfo(name="conv", opts_to_apply=opts))
 
 def dm_conv_172():
-  c0 = UOp(Ops.PARAM, dtypes.imageh((1, 240, 4)), (), 0)
+  c0 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((1, 240, 4)), (), 0)
   c2 = UOp.range(960, 4, AxisType.LOOP)
-  c5 = UOp(Ops.PARAM, dtypes.imageh((8, 384, 4)), (), 1)
+  c5 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((8, 384, 4)), (), 1)
   c7 = UOp.range(32, 0, AxisType.REDUCE)
   c10 = UOp.range(4, 1, AxisType.REDUCE)
   c13 = UOp.range(12, 3, AxisType.REDUCE)
   c18 = UOp.range(8, 2, AxisType.REDUCE)
-  c23 = UOp(Ops.PARAM, dtypes.imageh((240, 128, 4)), (), 2)
+  c23 = UOp(Ops.DEFINE_GLOBAL, dtypes.imageh((240, 128, 4)), (), 2)
   c35 = c5.index((c7*4+c10+c13*128+c18*1536))*c23.index((c10*4+c2%4+c7*16+c2//4*512))
-  c37 = UOp(Ops.PARAM, dtypes.float.ptr(960), (), 3)
+  c37 = UOp(Ops.DEFINE_GLOBAL, dtypes.float.ptr(960), (), 3)
   c39 = c35.reduce(c7, c10, arg=Ops.ADD)+c37.index(c2)
   c50 = (1.0+((c39+0.044708251953125*(c39*(c39*c39)))*-2.3021129851685216).exp2()).reciprocal()*c39
   c53 = c50.reduce(c18, c13, arg=Ops.ADD)*0.010416666666666666
@@ -91,7 +91,7 @@ allocator = Device.default.allocator
 ps = get_program(ast, renderer)
 cr = CompiledRunner(replace(ps, device=Device.DEFAULT))
 
-gs = sorted(dedup([u for u in ast.toposort() if u.op is Ops.PARAM]), key=lambda u: u.arg)
+gs = sorted(dedup([u for u in ast.toposort() if u.op is Ops.DEFINE_GLOBAL]), key=lambda u: u.arg)
 # print(len(gs))
 # print([g.dtype for g in gs])
 bufs = [Buffer(ps.device, g.size, g.dtype if isinstance(g.dtype, ImageDType) else g.dtype._base).ensure_allocated() for g in gs]

test/external/external_fuzz_qcom_cpu_cache.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-from tinygrad.tensor import Tensor
-import numpy as np
-
-while True:
-  arr = np.ones(1000000, dtype=np.uint8)
-  print(f"numpy: {(arr + 1)[:10]}")
-
-  ptr = arr.ctypes.data
-  tensor = Tensor.from_blob(ptr, arr.shape, dtype='uint8', device='QCOM').realize() + 1
-  print(f"from_blob: {tensor.numpy()[:10]}")

test/external/external_test_amd.py

@@ -20,7 +20,7 @@ class TestAMD(unittest.TestCase):
                               global_size=TestAMD.d0_runner.global_size, local_size=TestAMD.d0_runner.local_size)
       TestAMD.d0_runner.clprg(TestAMD.a.uop.buffer._buf, TestAMD.b.uop.buffer._buf,
                               global_size=TestAMD.d0_runner.global_size, local_size=TestAMD.d0_runner.local_size)
-    val = TestAMD.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestAMD.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 4000.0, f"got val {val}"
 
 if __name__ == "__main__":

test/external/external_test_hcq.py

@@ -65,7 +65,7 @@ class TestHCQ(unittest.TestCase):
     q.submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 2000.0, f"got val {val}"
 
   def test_run_1000_times(self):
@@ -81,7 +81,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.compute_queue().signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
       TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 2000.0, f"got val {val}"
 
   def test_run_to_3(self):
@@ -95,7 +95,7 @@ class TestHCQ(unittest.TestCase):
     q.signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 3.0, f"got val {val}"
 
   def test_update_exec(self):
@@ -106,9 +106,9 @@ class TestHCQ(unittest.TestCase):
     q.signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 0.0, f"got val {val}, should not be updated"
 
   @unittest.skipUnless(Device.DEFAULT == "NV", "Only NV supports bind")
@@ -126,7 +126,7 @@ class TestHCQ(unittest.TestCase):
       TestHCQ.compute_queue().signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
       TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 2000.0, f"got val {val}"
 
   @unittest.skipUnless(Device.DEFAULT == "NV", "Only NV supports bind")
@@ -141,9 +141,9 @@ class TestHCQ(unittest.TestCase):
     q.submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 0.0, f"got val {val}, should not be updated"
 
   @unittest.skipIf(CI, "Can't handle async update on CPU")
@@ -174,7 +174,7 @@ class TestHCQ(unittest.TestCase):
     q.signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
 
   def test_submit_empty_queues(self):
@@ -206,7 +206,7 @@ class TestHCQ(unittest.TestCase):
     q.submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
 
   def test_copy_1000_times(self):
@@ -221,7 +221,7 @@ class TestHCQ(unittest.TestCase):
     # confirm the signal didn't exceed the put value
     with self.assertRaises(RuntimeError):
       TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value + 1, timeout=50)
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 0.0, f"got val {val}"
 
   def test_copy(self):
@@ -231,7 +231,7 @@ class TestHCQ(unittest.TestCase):
     q.submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 1.0, f"got val {val}"
 
   @unittest.skipUnless(Device.DEFAULT == "NV", "Only NV supports bind")
@@ -248,7 +248,7 @@ class TestHCQ(unittest.TestCase):
     # confirm the signal didn't exceed the put value
     with self.assertRaises(RuntimeError):
       TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value + 1, timeout=50)
-    val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[1]
+    val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[1]
     assert val == 0.0, f"got val {val}"
 
   def test_copy_bandwidth(self):
@@ -288,7 +288,7 @@ class TestHCQ(unittest.TestCase):
     q.submit(TestHCQ.d0)
     TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
     TestHCQ.d0.timeline_value += 1
-    val = TestHCQ.a.uop.buffer.as_memoryview().cast("f")[0]
+    val = TestHCQ.a.uop.buffer.as_buffer().cast("f")[0]
     assert val == 1.0, f"got val {val}"
 
   def test_cross_device_signal(self):
@@ -319,7 +319,7 @@ class TestHCQ(unittest.TestCase):
       q.signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value).submit(TestHCQ.d0)
       TestHCQ.d0._wait_signal(TestHCQ.d0.timeline_signal, TestHCQ.d0.timeline_value)
       TestHCQ.d0.timeline_value += 1
-      val = TestHCQ.b.uop.buffer.as_memoryview().cast("f")[0]
+      val = TestHCQ.b.uop.buffer.as_buffer().cast("f")[0]
       assert val == 1.0, f"got val {val}"
 
 if __name__ == "__main__":

test/external/fuzz_graph.py

@@ -30,7 +30,7 @@ def alloc_rawbuffer(device, fill=False):
   if fill:
     with Context(DEBUG=0):
       data = np.random.randint(-10000, 10000, size=rawbuf.size, dtype=_to_np_dtype(rawbuf.dtype))
-      rawbuf.copyin(Tensor(data).realize().uop.base.realized.as_memoryview())
+      rawbuf.copyin(Tensor(data).realize().uop.base.realized.as_buffer())
   return rawbuf
 
 def gen_kernel_ji(device, deps):
@@ -93,7 +93,7 @@ def run_jit(jis, all_buffers, input_buffers, var_vals):
 
   with Context(DEBUG=0):
     res_buffers = []
-    for rawbuf in all_buffers: res_buffers.append(rawbuf.as_memoryview())
+    for rawbuf in all_buffers: res_buffers.append(rawbuf.as_buffer())
     return res_buffers
 
 def fuzz_graph(jis, all_buffers, input_buffers):

test/external/fuzz_linearizer.py

@@ -0,0 +1,339 @@
+import random, traceback, ctypes, argparse, os
+from typing import Any
+import numpy as np
+from collections import defaultdict
+from extra.optimization.helpers import load_worlds, ast_str_to_lin, kern_str_to_lin
+from tinygrad.engine.realize import get_program
+
+# We need to insert ioctl before opening devices.
+if os.getenv("VALIDATE_HCQ", 0) != 0:
+  try:
+    import extra.nv_gpu_driver.nv_ioctl
+    from tinygrad import Device
+    _, _ = Device["NV"], Device["CUDA"]
+  except Exception: pass
+
+  try:
+    import extra.qcom_gpu_driver.opencl_ioctl
+    from tinygrad import Device
+    _, _ = Device["QCOM"], Device["CL"]
+  except Exception: pass
+
+from tinygrad import Tensor, Device, dtypes
+from tinygrad.tensor import _to_np_dtype
+from tinygrad.codegen.opt.kernel import Kernel
+from tinygrad.codegen.opt import Opt, OptOps
+from tinygrad.codegen.opt.search import get_kernel_actions, bufs_from_lin
+from tinygrad.engine.realize import CompiledRunner
+from tinygrad.helpers import getenv, from_mv, prod, colored, Context, DEBUG, Timing
+from tinygrad.uop.ops import UOp, Ops
+from tinygrad.device import is_dtype_supported
+
+def on_linearizer_will_run(): pass
+def on_linearizer_did_run(): pass
+def compare_states(x, y): return (True, "")
+
+if getenv("VALIDATE_HCQ"):
+  if Device.DEFAULT == "NV":
+    print("VALIDATE_HCQ: Comparing NV to CUDA")
+    import extra.nv_gpu_driver.nv_ioctl
+    validate_device = Device["CUDA"]
+    on_linearizer_will_run = extra.nv_gpu_driver.nv_ioctl.before_launch
+    on_linearizer_did_run = extra.nv_gpu_driver.nv_ioctl.collect_last_launch_state
+    compare_states = extra.nv_gpu_driver.nv_ioctl.compare_launch_state
+  elif Device.DEFAULT == "QCOM":
+    print("VALIDATE_HCQ: Comparing QCOM to CL")
+    import extra.qcom_gpu_driver.opencl_ioctl
+    validate_device = Device["CL"]
+    on_linearizer_will_run = extra.qcom_gpu_driver.opencl_ioctl.before_launch
+    on_linearizer_did_run = extra.qcom_gpu_driver.opencl_ioctl.collect_last_launch_state
+    compare_states = extra.qcom_gpu_driver.opencl_ioctl.compare_launch_state
+  else:
+    print(colored("VALIDATE_HCQ options is ignored", 'red'))
+
+def tuplize_uops(uops:list[UOp]) -> tuple:
+  return tuple([(x.op, x.dtype, tuple(uops.index(x) for x in x.src), x.arg) for x in uops])
+
+def get_fuzz_rawbufs(lin):
+  rawbufs = bufs_from_lin(lin)
+
+  # Reallocate output buffer with additional area to detect out-of-bounds writes.
+  RED_AREA_SIZE = 1024
+  # setting output  # TODO: multi-output kernel
+  rawbufs[0] = get_fuzz_rawbuf_like(rawbufs[0], zero=True, size=rawbufs[0].size+RED_AREA_SIZE)
+  # setting inputs
+  with Context(DEBUG=0):
+    for rawbuf in rawbufs[1:]:
+      if dtypes.is_unsigned(rawbuf.dtype):
+        data = np.random.randint(0, 100, size=rawbuf.size, dtype=_to_np_dtype(rawbuf.dtype))
+      elif dtypes.is_int(rawbuf.dtype):
+        data = np.random.randint(-100, 100, size=rawbuf.size, dtype=_to_np_dtype(rawbuf.dtype))
+      elif rawbuf.dtype == dtypes.bool:
+        data = np.random.choice([True, False], size=rawbuf.size)
+      elif rawbuf.dtype == dtypes.half:
+        data = np.random.uniform(-1, 1, size=rawbuf.size).astype(dtype=_to_np_dtype(rawbuf.dtype))
+      else:
+        data = np.random.uniform(-10, 10, size=rawbuf.size).astype(dtype=_to_np_dtype(rawbuf.dtype))
+      rawbuf.copyin(Tensor(data, device=lin.opts.device).realize().uop.base.realized.as_buffer())
+  return rawbufs
+
+def get_fuzz_rawbuf_like(old_rawbuf, zero=False, copy=False, size=None, force_device=None):
+  rawbuf = type(old_rawbuf)(force_device or old_rawbuf.device, old_rawbuf.size if size is None else size, old_rawbuf.dtype).allocate()
+  if copy:
+    with Context(DEBUG=0): rawbuf.copyin(old_rawbuf.as_buffer())
+  elif zero:
+    with Context(DEBUG=0):
+      mv = memoryview(bytearray(rawbuf.size * rawbuf.dtype.itemsize))
+      ctypes.memset(from_mv(mv), 0, len(mv))
+      rawbuf.copyin(mv)
+  return rawbuf
+
+def run_linearizer(lin: Kernel, rawbufs=None, var_vals=None) -> tuple[str, Any]: # (error msg, run state)
+  if rawbufs is None: rawbufs = bufs_from_lin(lin)
+  if var_vals is None: var_vals = {v.expr: v.min for v in lin.vars}
+
+  # TODO: images needs required_optimization
+  try:
+    prg = CompiledRunner(get_program(lin.get_optimized_ast(), lin.opts))
+  except KeyboardInterrupt: raise
+  except Exception:
+    traceback.print_exc()
+    return "COMPILE_ERROR", None
+
+  if getenv("VALIDATE_HCQ"): on_linearizer_will_run()
+  try:
+    prg(rawbufs, var_vals, wait=True)
+  except KeyboardInterrupt: raise
+  except Exception:
+    traceback.print_exc()
+    return "EXEC_ERROR", None
+
+  if getenv("VALIDATE_HCQ"): run_state = on_linearizer_did_run()
+  else: run_state = None
+
+  return "PASS", run_state
+
+def compare_linearizer(lin: Kernel, rawbufs=None, var_vals=None, ground_truth=None, rtol=1e-2, atol=1e-2):
+  # TODO: for bfloat16 it compiles linearizer, but it does not run because numpy cannot generate bf16 buffer.
+  has_bf16 = any(b.dtype.base == dtypes.bfloat16 for b in lin.bufs)
+
+  # TODO: raise specific fuzzing errors instead of str, and propagate the error message
+  try:
+    if rawbufs is None:
+      rawbufs = get_fuzz_rawbufs(lin)
+    else:
+      rawbufs[0] = get_fuzz_rawbuf_like(rawbufs[0], zero=True) # get a new output buffer
+  except KeyboardInterrupt: raise
+  except BaseException:
+    return ("RAWBUFS_ERROR", rawbufs, var_vals, ground_truth, None)
+
+  if var_vals is None:
+    # TODO: handle symbolic max case
+    var_vals = {v.expr: random.randint(v.vmin, v.vmax) for v in lin.ast.variables()}
+
+  if ground_truth is None and not has_bf16:
+    unoptimized = Kernel(lin.ast)
+    if run_linearizer(unoptimized, rawbufs, var_vals)[0] != "PASS":
+      return ("BASELINE_ERROR", rawbufs, var_vals, ground_truth, None)
+    ground_truth = np.frombuffer(rawbufs[0].as_buffer(), _to_np_dtype(rawbufs[0].dtype)).copy()
+
+  rawbufs[0] = get_fuzz_rawbuf_like(rawbufs[0], zero=True) # get a new output buffer
+  run_msg, run_state = run_linearizer(lin, rawbufs, var_vals)
+  if run_msg != "PASS": return (run_msg, rawbufs, var_vals, ground_truth, run_state)
+
+  try:
+    if not has_bf16:
+      result = np.frombuffer(rawbufs[0].as_buffer(), _to_np_dtype(rawbufs[0].dtype))
+      np.testing.assert_allclose(result, ground_truth, rtol=rtol, atol=atol)
+  except KeyboardInterrupt: raise
+  except AssertionError as e:
+    if DEBUG >= 2:
+      print(f"COMPARE_ERROR details: {e}")
+      if getenv("DEBUG_VALUES") > 0:
+        mismatch_indices = np.where(~np.isclose(result, ground_truth, rtol=rtol, atol=atol))
+        mismatched_result = result[mismatch_indices]
+        mismatched_ground_truth = ground_truth[mismatch_indices]
+        for i, idx in enumerate(mismatch_indices[0]):
+          print(f"mismatch at {idx=}: result={mismatched_result[i]} <> ground_truth={mismatched_ground_truth[i]}")
+    return ("COMPARE_ERROR", rawbufs, var_vals, ground_truth, run_state)
+
+  return ("PASS", rawbufs, var_vals, ground_truth, run_state)
+
+def fuzz_linearizer(lin: Kernel, rtol=1e-2, atol=1e-2, opts_list=None):
+  SEED = getenv("SEED", 42)
+  random.seed(SEED)
+  np.random.seed(SEED)
+  print(lin.ast)
+  print(lin.colored_shape())
+  seen_uops = {}
+  last_lins = [lin]
+  failures:defaultdict[str, list[tuple[tuple[UOp, ...], list[Opt]]]] = defaultdict(list)
+  rawbufs, var_vals, ground_truth, validate_rawbufs = None, None, None, None
+
+  FUZZ_ALL_ACTIONS = getenv("FUZZ_ALL_ACTIONS", 0)
+  FUZZ_MAX_SIZE = getenv("FUZZ_MAX_SIZE", 0)
+  FUZZ_IGNORE_SIMPLE_OPS = getenv("FUZZ_IGNORE_SIMPLE_OPS", 1)
+
+  if FUZZ_MAX_SIZE > 0 and prod(lin.full_shape) > FUZZ_MAX_SIZE:
+    print("skipping large kernel")
+    return failures
+  if FUZZ_IGNORE_SIMPLE_OPS and _is_simple(lin):
+    print("skipping simple kernel")
+    return failures
+
+  test_depth = 1 if opts_list is not None else getenv("DEPTH", 1 if FUZZ_ALL_ACTIONS else 10)
+  for depth in range(test_depth):
+    next_lins = []
+    for lin in last_lins:
+      if opts_list is None: actions = get_kernel_actions(lin, include_0=False)
+      else:
+        actions = {}
+        for oi,opts in enumerate(opts_list):
+          lin2 = lin.copy()
+          for o in opts: lin2.apply_opt(o)
+          actions[oi] = lin2
+
+      if not actions: continue
+      if depth == 0 and getenv("FUZZ_REQUIRE_TC", 0):
+        tc_acts = {i: k for k in actions.values() if k.applied_opts[0].op == OptOps.TC}
+        if len(tc_acts) == 0: return failures
+        else: actions = tc_acts
+
+      test_lins = list(actions.values())
+      if FUZZ_ALL_ACTIONS: print(f"testing {lin.applied_opts=} with {len(actions)} actions")
+      elif opts_list is None: test_lins = [random.choice(test_lins)]
+
+      for test_lin in test_lins:
+        if not FUZZ_ALL_ACTIONS and test_lin.applied_opts: print(f"applied opts: {test_lin.applied_opts}")
+
+        # stop if kernel uops repeat
+        try: tuops = tuplize_uops(get_program(test_lin.get_optimized_ast(), test_lin.ren).uops)
+        except KeyboardInterrupt: raise
+        except BaseException as e:
+          print(test_lin.ast)
+          print(test_lin.applied_opts)
+          print(e)
+          failures["LINEARIZE_ERROR"].append((test_lin.ast, test_lin.applied_opts))
+          continue
+
+        if tuops in seen_uops: continue
+        seen_uops[tuops] = tuple(test_lin.applied_opts)
+
+        if not FUZZ_ALL_ACTIONS: print(test_lin.colored_shape())
+
+        (msg, rawbufs, var_vals, ground_truth, state1) = compare_linearizer(test_lin, rawbufs, var_vals, ground_truth, rtol=rtol, atol=atol)
+        if state1 is not None and validate_device is not None:
+          validate_lin = test_lin.copy()
+          validate_lin.ren = validate_device.renderer
+          if validate_rawbufs is None:
+            validate_rawbufs = [get_fuzz_rawbuf_like(x, copy=True, force_device=validate_device.device) for x in rawbufs]
+          (_msg, _, _, _, state2) = compare_linearizer(validate_lin, validate_rawbufs, var_vals, ground_truth, rtol=rtol, atol=atol)
+
+          if _msg != "PASS": failures[f"VALIDATE_DEV_{_msg}"].append((validate_lin.ast, validate_lin.applied_opts))
+
+          ok, err_msg = compare_states(state1, state2)
+          if not ok: failures["HCQ_COMPARE_FAILURE"].append((err_msg, test_lin.ast, test_lin.applied_opts, state1, state2))
+
+        if msg != "PASS":
+          print(test_lin.ast)
+          print(test_lin.applied_opts)
+          print(msg)
+          failures[msg].append((test_lin.ast, test_lin.applied_opts))
+          continue
+
+        next_lins.append(test_lin)
+
+    last_lins = next_lins
+    if FUZZ_ALL_ACTIONS: print(f"depth={depth} total_lins={len(last_lins)} {failures=}")
+  return failures
+
+def _is_simple(lin: Kernel) -> bool:
+  if len(lin.ast.src) > 1: return False
+  ast:UOp = lin.ast.src[0]
+  if ast.src[0].op is Ops.CAST and ast.src[0].src[0].op is Ops.LOAD: return True
+  return False
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Run a fuzz testing on one or more kernels", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument("--ast", type=str, default=None, help="the ast for the kernel to be optimized")
+  parser.add_argument("--file", type=str, default=None, help="a file containing asts to be optimized, one per line")
+  parser.add_argument("--beamreplay", type=str, default=None, help="replay asts and opts got from beam with CAPTURE_BEAM")
+  parser.add_argument("--logfile", type=str, default=None, help="a file containing a tuple of ast and applied_opts, one per line")
+  parser.add_argument("--expected-failures", type=int, default=0, help="the number of expected failed kernels")
+  parser.add_argument("--rtol", type=float, default=1e-2, help="relative tolerance for numerical comparison")
+  parser.add_argument("--atol", type=float, default=1e-2, help="absolute tolerance for numerical comparison")
+  args = parser.parse_args()
+
+  opts_list = None
+  if args.ast is not None:
+    print("loaded AST from CLI")
+    ast_strs = [args.ast]
+  elif args.file is not None:
+    print(f"loading ASTs from file '{args.file}'")
+    with open(args.file, 'r') as file:
+      ast_strs = file.readlines()
+  elif args.beamreplay is not None:
+    print(f"loading BEAM replay from file '{args.beamreplay}'")
+    with open(args.beamreplay, 'r') as file: fdata = file.readlines()
+    ast_strs, opts_list = [x.split(' :: ')[0] for x in fdata if not x.startswith("#")], [x.split(' :: ')[1] for x in fdata if not x.startswith("#")]
+
+    # dedup ast_strs and opts_list
+    dct = defaultdict(list)
+    for i in range(len(ast_strs)): dct[ast_strs[i]].append(eval(opts_list[i]))
+    ast_strs_items = list(dct.keys())
+    opts_list = [dct[c] for c in ast_strs_items]
+  elif args.logfile is not None:
+    print(f"loading ASTs from LOGKERNS file '{args.file}'")
+    with open(args.logfile, 'r') as file:
+      kern_strs = file.readlines()
+      test_lins = [kern_str_to_lin(kern_str) for kern_str in kern_strs]
+      ast_strs = [f"{lin.ast}" for lin in test_lins]
+  else:
+    print("loading ASTs from world")
+    ast_strs = load_worlds(filter_reduce=False, filter_novariable=False)
+
+  print(f"{len(ast_strs)=}")
+  tested = 0
+  failed_ids = []
+  failures = defaultdict(list)
+  seen_ast_strs = set()
+
+  try:
+    for i, ast in enumerate(ast_strs[:getenv("FUZZ_N", len(ast_strs))]):
+      if (nth := getenv("FUZZ_NTH", -1)) != -1 and i != nth: continue
+      if getenv("FUZZ_IMAGEONLY") and "dtypes.image" not in ast: continue
+      if "dtypes.image" in ast and Device.DEFAULT not in {"CL", "QCOM"}: continue  # IMAGE is only for CL
+      if ast in seen_ast_strs: continue
+      seen_ast_strs.add(ast)
+
+      lin = ast_str_to_lin(ast)
+      if not all(is_dtype_supported(buf.dtype) for buf in lin.bufs):
+        print("skipping kernel due to not supported dtype")
+        continue
+
+      with Timing(f"tested ast {i}: "):
+        tested += 1
+        fuzz_failures = fuzz_linearizer(lin, rtol=args.rtol, atol=args.atol, opts_list=(opts_list[i] if opts_list else None))
+        if fuzz_failures: failed_ids.append(i)
+        for k, v in fuzz_failures.items():
+          for f in v:
+            failures[k].append(f)
+  except KeyboardInterrupt: print(colored("STOPPING...", 'red'))
+
+  for msg, errors in failures.items():
+    for i, payload in enumerate(errors):
+      print(f"{msg} {i} kernel: {payload}") # easier to use with output with verify_kernel.py
+
+  print(f"{tested=}")
+  if failures:
+    print(f"{failed_ids=}")
+    for msg, errors in failures.items():
+      print(f"{msg}: {len(errors)}")
+    if len(failed_ids) == args.expected_failures:
+      print(colored(f"{len(failed_ids)} failed as expected", "yellow"))
+  if len(failed_ids) != args.expected_failures:
+    print(colored(f"failed on {len(failed_ids)} kernels, expected {args.expected_failures}", "red"))
+    # TODO: fix this
+    # raise RuntimeError(f"failed on {len(failed_ids)} kernels, expected {args.expected_failures}")
+  else:
+    print(colored("all passed", "green"))

test/external/speed_compare_amd_am.py

@@ -0,0 +1,157 @@
+from tinygrad import Device, dtypes
+from tinygrad.helpers import getenv, colorize_float, DEBUG
+from extra.optimization.helpers import load_worlds, ast_str_to_lin
+from test.external.fuzz_linearizer import get_fuzz_rawbufs
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from tinygrad.codegen.opt.search import bufs_from_lin
+from tinygrad.engine.realize import CompiledRunner, get_program
+from tinygrad.tensor import _to_np_dtype
+from tinygrad.runtime.ops_amd import AMDDevice
+from contextlib import contextmanager
+import numpy as np
+import os, random, statistics
+
+am_signal_pages, am_signal_pool, am_devices = [], [], []
+amd_signal_pages, amd_signal_pool, amd_devices = [], [], []
+
+def rebind_vfio(pcibus="0000:44:00.0"):
+  print("rebind ", pcibus)
+  os.system("sudo rmmod amdgpu")
+  os.system("sudo modprobe vfio-pci")
+
+  base = f"/sys/bus/pci/devices/{pcibus}"
+  if os.path.exists(f"{base}/driver"):
+    with open(f"{base}/driver/unbind", "w") as f: f.write(pcibus)
+  with open(f"{base}/driver_override", "w") as f: f.write("vfio-pci")
+  with open("/sys/bus/pci/drivers_probe", "w") as f: f.write(pcibus)
+
+  os.system("sudo modprobe amdgpu")
+  os.system("rocm-smi --setprofile compute")
+  os.system("rocm-smi --setperflevel high")
+
+@contextmanager
+def run_amd():
+  global amd_signal_pages, amd_signal_pool, amd_devices
+  AMDDevice.driverless = False
+  AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices = amd_signal_pages, amd_signal_pool, amd_devices
+  yield
+  amd_signal_pages, amd_signal_pool, amd_devices = AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices
+  AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices = [], [], []
+
+@contextmanager
+def run_am():
+  global am_signal_pages, am_signal_pool, am_devices
+  AMDDevice.driverless = True
+  AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices = am_signal_pages, am_signal_pool, am_devices
+  yield
+  am_signal_pages, am_signal_pool, am_devices = AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices
+  AMDDevice.signal_pages, AMDDevice.signal_pool, AMDDevice.devices = [], [], []
+
+if __name__ == "__main__":
+  CHECK_CPU = getenv("CHECK_CPU", 0)
+  SEED = getenv("SEED", 42)
+  CNT = getenv("CNT", 7)
+  random.seed(SEED)
+  np.random.seed(SEED)
+
+  # TODO: NUM=780 is super slow
+  # NUM=1907 is broken on AMD and AM have some mismatches (0 vs 1)
+  # kfd feels so bad when taking gpu out while it's running... Need hacks to rebind it before running.
+  rebind_vfio(pcibus="0000:44:00.0")
+
+  ast_strs = load_worlds(filter_reduce=False, filter_novariable=True)
+
+  with run_am():
+    amdev = Device["AMD:1"]
+
+  with run_amd():
+    amddev = Device["AMD"]
+
+  if CHECK_CPU: cpudev = Device["CPU"]
+
+  single = getenv("NUM", -1)
+  if single != -1: ast_strs = ast_strs[single:single+1]
+
+  average_tm_amd, average_tm_am = 0, 0
+  for num,ast in enumerate(ast_strs):
+    with run_amd():
+      amdlin = ast_str_to_lin(ast, opts=amddev.renderer)
+      amdlin.apply_opts(hand_coded_optimizations(amdlin))
+      has_bf16 = any(b.dtype == dtypes.bfloat16 for b in amdlin.bufs)
+
+      amd_prg = CompiledRunner(get_program(amdlin.get_optimized_ast(), amdlin.opts))
+      amdbufs = bufs_from_lin(amdlin)
+      test_amdbufs = get_fuzz_rawbufs(amdlin) if not has_bf16 else amdbufs
+      if not has_bf16: contents = [buf.as_buffer() for buf in test_amdbufs]
+
+    with run_am():
+      rdr = amdev.renderer
+      rdr.device = "AMD:1"
+      amlin = ast_str_to_lin(ast, opts=amdev.renderer)
+      amlin.apply_opts(hand_coded_optimizations(amlin))
+      am_prg = CompiledRunner(get_program(amlin.get_optimized_ast(), amlin.opts))
+      ambufs = bufs_from_lin(amlin)
+      test_ambufs = get_fuzz_rawbufs(amlin) if not has_bf16 else ambufs
+      if not has_bf16:
+        for i,rawbuf in enumerate(test_ambufs): rawbuf.copyin(contents[i])
+
+    if CHECK_CPU:
+      cpu_rdr = cpudev.renderer
+      cpu_rdr.device = "CPU"
+      cpulin = ast_str_to_lin(ast, opts=cpu_rdr)
+      cpulin.apply_opts(hand_coded_optimizations(cpulin))
+      cpu_prg = CompiledRunner(get_program(cpulin.get_optimized_ast(), cpulin.opts))
+      cpubufs = bufs_from_lin(cpulin)
+      test_cpubufs = get_fuzz_rawbufs(cpulin) if not has_bf16 else ambufs
+      if not has_bf16:
+        for i,rawbuf in enumerate(test_cpubufs): rawbuf.copyin(contents[i])
+
+    # warmup
+    tm_amd, tm_am, failed = [], [], False
+    with run_amd():
+      try:
+        amd_prg(test_amdbufs, {}, wait=True)
+        for i in range(CNT): tm_amd.append(amd_prg(amdbufs, {}, wait=True))
+      except RuntimeError:
+        print("AMD FAILED")
+        tm_amd = [1e9]
+        failed = True
+
+    with run_am():
+      try:
+        am_prg(test_ambufs, {}, wait=True)
+        for i in range(CNT): tm_am.append(am_prg(ambufs, {}, wait=True))
+      except RuntimeError:
+        print("AM FAILED")
+        tm_am = [1e9]
+        failed = True
+
+    if CHECK_CPU:
+      cpu_prg(test_cpubufs, {}, wait=True)
+      for i in range(1): cpu_prg(cpubufs, {}, wait=True)
+
+    if not failed and not has_bf16:
+      with run_amd():
+        curesult = np.frombuffer(test_amdbufs[0].as_buffer(), _to_np_dtype(test_amdbufs[0].dtype))
+
+      with run_am():
+        amresult = np.frombuffer(test_ambufs[0].as_buffer(), _to_np_dtype(test_ambufs[0].dtype))
+
+      if CHECK_CPU:
+        cpuresult = np.frombuffer(test_cpubufs[0].as_buffer(), _to_np_dtype(test_cpubufs[0].dtype))
+        np.testing.assert_allclose(amresult, cpuresult, rtol=1e-2, atol=1e-2)
+        np.testing.assert_allclose(curesult, cpuresult, rtol=1e-2, atol=1e-2)
+
+      try:
+        np.testing.assert_allclose(curesult, amresult, rtol=1e-2, atol=1e-2)
+      except AssertionError as e:
+        print("AM and AMD results do not match")
+        print(e)
+
+    bam = statistics.median(tm_am)
+    bamd = statistics.median(tm_amd)
+    average_tm_amd += bamd
+    average_tm_am += bam
+    ratio = bam/bamd
+    print(f"{average_tm_am/average_tm_amd:5.2f}x -- {num:4d} {colorize_float(ratio)} {bam*1e6:7.2f} vs {bamd*1e6:7.2f} us", amlin.name)
+    if DEBUG > 3 and ratio > 1.04: print(f"AM slower {ratio}", amlin.ast, amlin.applied_opts)

test/external/speed_compare_cuda_nv.py

@@ -0,0 +1,69 @@
+from tinygrad import Device, dtypes
+from tinygrad.helpers import getenv, colorize_float
+from extra.optimization.helpers import load_worlds, ast_str_to_lin
+from test.external.fuzz_linearizer import get_fuzz_rawbufs
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from tinygrad.codegen.opt.search import bufs_from_lin
+from tinygrad.engine.realize import CompiledRunner, get_program
+from tinygrad.tensor import _to_np_dtype
+import numpy as np
+
+if __name__ == "__main__":
+  ast_strs = load_worlds(filter_reduce=False, filter_novariable=True)
+  cudev = Device["CUDA"]
+  nvdev = Device["NV"]
+
+  # NUM=112 python3 test/external/speed_compare_cuda_nv.py
+
+  single = getenv("NUM", -1)
+  if single != -1: ast_strs = ast_strs[single:single+1]
+
+  average_tm_cuda, average_tm_nv = 0, 0
+  for num,ast in enumerate(ast_strs):
+    # cuda compile
+    culin = ast_str_to_lin(ast, opts=cudev.renderer)
+    culin.apply_opts(hand_coded_optimizations(culin))
+    has_bf16 = any(b.dtype == dtypes.bfloat16 for b in culin.bufs)
+
+    cuda_prg = CompiledRunner(get_program(culin.get_optimized_ast(), culin.opts))
+    cubufs = bufs_from_lin(culin)
+    test_cubufs = get_fuzz_rawbufs(culin) if not has_bf16 else cubufs
+
+    rdr = nvdev.renderer
+    rdr.device = "NV"
+    nvlin = ast_str_to_lin(ast, opts=rdr)
+    nvlin.apply_opts(hand_coded_optimizations(nvlin))
+    nv_prg = CompiledRunner(get_program(nvlin.get_optimized_ast(), nvlin.opts))
+    nvbufs = bufs_from_lin(nvlin)
+    test_nvbufs = get_fuzz_rawbufs(nvlin) if not has_bf16 else nvbufs
+    if not has_bf16:
+      for i,rawbuf in enumerate(test_nvbufs): rawbuf.copyin(test_cubufs[i].as_buffer())
+
+    # warmup
+    tm_cuda, tm_nv, failed = [], [], False
+    try:
+      cuda_prg(test_cubufs, {}, wait=True)
+      for i in range(5): tm_cuda.append(cuda_prg(cubufs, {}, wait=True))
+    except RuntimeError:
+      print("CUDA FAILED")
+      tm_cuda = [1e9]
+      failed = True
+
+    try:
+      nv_prg(test_nvbufs, {}, wait=True)
+      for i in range(5): tm_nv.append(nv_prg(nvbufs, {}, wait=True))
+    except RuntimeError:
+      print("NV FAILED")
+      tm_nv = [1e9]
+      failed = True
+
+    if not failed and not has_bf16:
+      curesult = np.frombuffer(test_cubufs[0].as_buffer(), _to_np_dtype(test_cubufs[0].dtype))
+      nvresult = np.frombuffer(test_nvbufs[0].as_buffer(), _to_np_dtype(test_nvbufs[0].dtype))
+      np.testing.assert_allclose(curesult, nvresult, rtol=1e-2, atol=1e-2)
+
+    average_tm_cuda += min(tm_cuda)
+    average_tm_nv += min(tm_nv)
+    ratio = min(tm_nv)/min(tm_cuda)
+    print(f"{average_tm_nv/average_tm_cuda:5.2f}x -- {num:4d} {colorize_float(ratio)}  {min(tm_nv)*1e6:7.2f} us", nvlin.name)
+    if ratio > 1.04: print(f"NV slower {ratio}", nvlin.ast, nvlin.applied_opts)

test/external/speed_compare_cuda_ptx.py

@@ -0,0 +1,60 @@
+import itertools
+from tinygrad import Device
+from tinygrad.engine.realize import CompiledRunner, get_program
+from tinygrad.codegen.opt.heuristic import hand_coded_optimizations
+from tinygrad.helpers import getenv, colorize_float
+from extra.optimization.helpers import load_worlds, ast_str_to_lin
+from tinygrad.codegen.opt.search import bufs_from_lin
+from tinygrad.runtime.ops_cuda import PTXCompiler, PTXRenderer, CUDACompiler
+
+if __name__ == "__main__":
+  ast_strs = load_worlds(filter_reduce=False, filter_novariable=True)
+  # no bfloat16 for ptx at the moment
+  ast_strs = [x for x in ast_strs if "dtypes.bfloat16" not in x]
+  dev = Device["CUDA"]
+  ptx = PTXRenderer(dev.arch)
+
+  # NUM=112 python3 test/external/speed_compare_cuda_ptx.py
+
+  single = getenv("NUM", -1)
+  if single != -1: ast_strs = ast_strs[single:single+1]
+
+  average_tm_cuda, average_tm_ptx = 0, 0
+  for num,ast in enumerate(ast_strs):
+    # cuda compile
+    dev.compiler = CUDACompiler(dev.arch)
+    lin = ast_str_to_lin(ast, opts=dev.renderer)
+    lin.apply_opts(hand_coded_optimizations(lin))
+    cuda_prg = CompiledRunner(get_program(lin.get_optimized_ast(), lin.opts))
+
+    bufs = bufs_from_lin(lin)
+
+    # ptx compile
+    dev.compiler = PTXCompiler(dev.arch)
+    lin = ast_str_to_lin(ast, opts=ptx)
+    lin.apply_opts(hand_coded_optimizations(lin))
+    ptx_prg = CompiledRunner(get_program(lin.get_optimized_ast(), lin.opts))
+
+    # warmup
+    try:
+      cuda_prg(bufs, {}, wait=True)
+    except RuntimeError:
+      print("cuda failed ast:", num)
+      continue
+    ptx_prg(bufs, {}, wait=True)
+
+    tm_cuda, tm_ptx = [], []
+    for i in range(5):
+      tm_cuda.append(cuda_prg(bufs, {}, wait=True))
+      tm_ptx.append(ptx_prg(bufs, {}, wait=True))
+    average_tm_cuda += min(tm_cuda)
+    average_tm_ptx += min(tm_ptx)
+    ratio = min(tm_ptx)/min(tm_cuda)
+    print(f"{average_tm_ptx/average_tm_cuda:5.2f}x -- {num:4d} {colorize_float(ratio)}  {min(tm_ptx)*1e6:7.2f} us", lin.name)
+    if ratio > 1.5:
+      def fix(x): return x.replace('\t', ' ').strip()
+      ll1, ll2 = cuda_prg.lib.decode().split('\n'), ptx_prg.lib.decode().split('\n')
+      if single != -1:
+        for ln, (l1, l2) in enumerate(itertools.zip_longest(ll1, ll2, fillvalue='')):
+          print(f"{ln:5d} | {fix(l1):80s} | {fix(l2):80s}")
+      print(len(ll1), len(ll2), "RATIO", ratio, "us", min(tm_ptx)*1e6)

test/external/verify_kernel.py

@@ -0,0 +1,78 @@
+import argparse
+from collections import defaultdict
+from extra.optimization.helpers import kern_str_to_lin, time_linearizer
+from test.external.fuzz_linearizer import compare_linearizer
+from tinygrad.helpers import colored
+from tinygrad.codegen.opt.kernel import Kernel
+
+# Use this with the LOGKERNS options to verify that all executed kernels are valid and evaluate to the same ground truth results
+
+# Example for GPT2:
+# 1) Run the model to log all kernels: `PYTHONPATH=. LOGKERNS=/tmp/gpt2_kerns.txt JIT=1 HALF=1 BEAM=2 CACHELEVEL=0 python3 examples/gpt2.py --count 10 --temperature 0 --timing`   # noqa: E501
+# 2) Validate the kernel correctness: `PYTHONPATH=. python3 ./test/external/verify_kernel.py --file /tmp/gpt2_kerns.txt`
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Verify the correctness of one or more kernel", formatter_class=argparse.ArgumentDefaultsHelpFormatter)    # noqa: E501
+  parser.add_argument("--kernel", type=str, default=None, help="a string of a tuple of (ast, applied_opts,)")
+  parser.add_argument("--file", type=str, default=None, help="a file containing a tuple of ast and applied_opts, one per line")
+  parser.add_argument("--pkl", type=str, default=None, help="a pickle file containing a single tuple of ast and applied_opts")
+  parser.add_argument("--rtol", type=float, default=1e-2, help="relative tolerance for numerical comparison")
+  parser.add_argument("--atol", type=float, default=1e-2, help="absolute tolerance for numerical comparison")
+  parser.add_argument("--timing", action='store_true', help="show final timing for the kernel")
+  parser.add_argument("--expected-failures", type=int, default=0, help="the number of expected failed kernels")
+  args = parser.parse_args()
+
+  if args.kernel is not None:
+    print("loading kernel from args")
+    test_lins = [kern_str_to_lin(args.kernel)]
+  elif args.file is not None:
+    print(f"loading kernel from file '{args.file}'")
+    with open(args.file, 'r') as file:
+      kern_strs = file.readlines()
+      test_lins = [kern_str_to_lin(kern_str) for kern_str in kern_strs]
+  elif args.pkl is not None:
+    print(f"loading kernel from pickle file '{args.file}'")
+    import pickle
+    with open(args.pkl, 'rb') as file:
+      (ast, applied_opts,) = pickle.load(file)
+      lin = Kernel(ast)
+      lin.apply_opts(applied_opts)
+      test_lins = [lin]
+
+  else:
+    raise RuntimeError("no kernel specified; use --kernel, --file, or --pkl options")
+
+  print(f"verifying {len(test_lins)} kernels")
+
+  failed_ids = []
+  failures = defaultdict(list)
+  for i, test_lin in enumerate(test_lins):
+    print(f"testing kernel {i}")
+    print(test_lin.ast)
+    print(test_lin.applied_opts)
+    unoptimized_lin = Kernel(test_lin.ast)
+    print(f"{unoptimized_lin.colored_shape()} -> {test_lin.colored_shape()}")
+    (msg,rb,vv,gt) = compare_linearizer(test_lin, None, None, None, rtol=args.rtol, atol=args.atol)
+    if msg != "PASS":
+      failed_ids.append(i)
+      failures[msg].append((test_lin.ast, test_lin.applied_opts))
+    if args.timing:
+      tm = time_linearizer(test_lin, rb, allow_test_size=False, cnt=10)
+      print(f"final time {tm*1e6:9.0f} us")
+
+  for msg, errors in failures.items():
+    for i, (ast, opts) in enumerate(errors):
+      print(f"{msg} {i} AST: {ast}")
+      print(f"{msg} {i} OPTS: {opts}\n")
+
+  print(f"tested {len(test_lins)} kernels")
+  if failures:
+    print(f"{failed_ids=}")
+    for msg, errors in failures.items():
+      print(f"{msg}: {len(errors)}")
+    if len(failed_ids) == args.expected_failures:
+      print(colored(f"{len(failed_ids)} failed as expected", "yellow"))
+  if len(failed_ids) != args.expected_failures:
+    raise RuntimeError(f"failed on {len(failed_ids)} kernels, expected {args.expected_failures}")
+  else:
+    print(colored("all passed", "green"))

test/helpers.py

@@ -61,7 +61,7 @@ def eval_uop(uop:UOp, inputs:list[tuple[DType, list[Any]]]|None=None):
   for buf_dt, data in inputs or []:
     bufs.append(buf:=allocator.alloc(len(data) * buf_dt.itemsize))
     allocator._copyin(buf, memoryview(struct.pack(str(len(data)) + (buf_dt.fmt or ""), *data)))
-  g = UOp(Ops.PARAM, uop.dtype.ptr(), arg=0, src=())
+  g = UOp(Ops.DEFINE_GLOBAL, uop.dtype.ptr(), arg=0, src=())
   prg = get_program(UOp.store(g.index(UOp.const(dtypes.int, 0)), uop).sink(), PythonRenderer())
   prog = PythonProgram("run", PythonCompiler().compile(prg.src))
   prog(out_buf:=allocator.alloc(uop.dtype.itemsize), *bufs)

test/mockgpu/amd/amddriver.py

@@ -4,7 +4,7 @@ import tinygrad.runtime.autogen.am.am as am
 import tinygrad.runtime.autogen.amdgpu_drm as amdgpu_drm
 from tinygrad.helpers import from_mv
 from test.mockgpu.driver import VirtDriver, VirtFileDesc, TextFileDesc, DirFileDesc, VirtFile
-from test.mockgpu.amd.amdgpu import AMDGPU, gpu_props, GFX_TARGET_VERSION, MOCKGPU_ARCH
+from test.mockgpu.amd.amdgpu import AMDGPU, gpu_props
 
 libc = ctypes.CDLL(ctypes.util.find_library("c"))
 libc.mmap.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_long]
@@ -90,30 +90,35 @@ class AMDDriver(VirtDriver):
   def _prepare_gpu(self, gpu_id):
     self.doorbells[gpu_id] = memoryview(bytearray(0x2000))
     self.gpus[gpu_id] = AMDGPU(gpu_id)
-    # IP versions: rdna3 = GC 11.0.0, NBIF 4.3.0; rdna4 = GC 12.0.0, NBIF 6.3.1
-    ip_versions = {"rdna3": {"gc": (11, 0, 0), "sdma": (6, 0, 0), "nbif": (4, 3, 0)},
-                   "rdna4": {"gc": (12, 0, 0), "sdma": (6, 0, 0), "nbif": (6, 3, 1)}}[MOCKGPU_ARCH]
-    def ip_discovery_files(hwid, ver, base_addr):
-      p = f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{hwid}/0'
-      return [VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{hwid}', functools.partial(DirFileDesc, child_names=['0'])),
-              VirtFile(f'{p}/major', functools.partial(TextFileDesc, text=str(ver[0]))),
-              VirtFile(f'{p}/minor', functools.partial(TextFileDesc, text=str(ver[1]))),
-              VirtFile(f'{p}/revision', functools.partial(TextFileDesc, text=str(ver[2]))),
-              VirtFile(f'{p}/base_addr', functools.partial(TextFileDesc, text=base_addr))]
     self.tracked_files += [
       VirtFile('/sys/module/amdgpu', functools.partial(TextFileDesc, text="1")),
       VirtFile('/sys/module/amdgpu/parameters/ppfeaturemask', functools.partial(TextFileDesc, text="0xffff3fff")),
       VirtFile(f'/sys/devices/virtual/kfd/kfd/topology/nodes/{gpu_id}', functools.partial(DirFileDesc, child_names=['gpu_id', 'properties'])),
       VirtFile(f'/sys/devices/virtual/kfd/kfd/topology/nodes/{gpu_id}/gpu_id', functools.partial(TextFileDesc, text=f"{gpu_id}")),
       VirtFile(f'/sys/devices/virtual/kfd/kfd/topology/nodes/{gpu_id}/properties',
-        functools.partial(TextFileDesc, text=gpu_props.format(drm_render_minor=gpu_id, gfx_target_version=GFX_TARGET_VERSION))),
+        functools.partial(TextFileDesc, text=gpu_props.format(drm_render_minor=gpu_id))),
       VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/power_dpm_force_performance_level',
                functools.partial(TextFileDesc, text='profile_standard\n')),
       VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0',
                functools.partial(DirFileDesc, child_names=[str(am.GC_HWID), str(am.SDMA0_HWID), str(am.NBIF_HWID)])),
-      *ip_discovery_files(am.GC_HWID, ip_versions["gc"], '0x00001260\n0x0000A000\n0x0001C000\n0x02402C00'),
-      *ip_discovery_files(am.SDMA0_HWID, ip_versions["sdma"], '0x00001260\n0x0000A000\n0x0001C000\n0x02402C00'),
-      *ip_discovery_files(am.NBIF_HWID, ip_versions["nbif"], '0x00000000\n0x00000014\n0x00000D20\n0x00010400\n0x0241B000\n0x04040000'),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}', functools.partial(DirFileDesc, child_names=['0'])),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/major', functools.partial(TextFileDesc, text='11')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/minor', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.GC_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00001260\n0x0000A000\n0x0001C000\n0x02402C00')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}', functools.partial(DirFileDesc, child_names=['0'])),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/major', functools.partial(TextFileDesc, text='6')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/minor', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.SDMA0_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00001260\n0x0000A000\n0x0001C000\n0x02402C00')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}', functools.partial(DirFileDesc, child_names=['0'])),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/major', functools.partial(TextFileDesc, text='4')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/minor', functools.partial(TextFileDesc, text='3')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/revision', functools.partial(TextFileDesc, text='0')),
+      VirtFile(f'/sys/class/drm/renderD{gpu_id}/device/ip_discovery/die/0/{am.NBIF_HWID}/0/base_addr',
+               functools.partial(TextFileDesc, text='0x00000000\n0x00000014\n0x00000D20\n0x00010400\n0x0241B000\n0x04040000')),
       VirtFile(f'/dev/dri/renderD{gpu_id}', functools.partial(DRMFileDesc, driver=self, gpu=f"{self.gpus[gpu_id]}")),
     ]
 

test/mockgpu/amd/amdgpu.py

@@ -1,11 +1,8 @@
 import ctypes, time
 from test.mockgpu.gpu import VirtGPU
 from test.mockgpu.helpers import _try_dlopen_remu
-from tinygrad.helpers import getbits, to_mv, getenv
+from tinygrad.helpers import getbits, to_mv
 from tinygrad.runtime.support import c
-
-MOCKGPU_ARCH = getenv("MOCKGPU_ARCH", "rdna3")
-GFX_TARGET_VERSION = {"rdna3": 110000, "rdna4": 120000}[MOCKGPU_ARCH]
 import tinygrad.runtime.autogen.amd_gpu as amd_gpu, tinygrad.runtime.autogen.am.pm4_nv as pm4
 
 SDMA_MAX_COPY_SIZE = 0x400000
@@ -197,11 +194,10 @@ class PM4Executor(AMDQueue):
     scratch_size = wavesize * 4  # This gives the scratch size per thread (lane)
 
     assert prg_sz > 0, "Invalid prg ptr (not found in mapped ranges)"
-    # Pass valid memory ranges, rsrc2, scratch_size and arch to Python emulator
+    # Pass valid memory ranges, rsrc2, and scratch_size to Python emulator
     if hasattr(remu, 'valid_mem_ranges'): remu.valid_mem_ranges = self.gpu.mapped_ranges
     if hasattr(remu, 'rsrc2'): remu.rsrc2 = rsrc2
     if hasattr(remu, 'scratch_size'): remu.scratch_size = scratch_size
-    if hasattr(remu, 'arch'): remu.arch = self.gpu.arch
     err = remu.run_asm(prg_addr, prg_sz, *gl, *lc, args_addr)
     if err != 0: raise RuntimeError("remu does not support the new instruction introduced in this kernel")
 
@@ -318,7 +314,6 @@ class AMDGPU(VirtGPU):
     self.regs = AMDGPURegisters()
     self.mapped_ranges = set()
     self.queues = []
-    self.arch = MOCKGPU_ARCH
 
   def map_range(self, vaddr, size): self.mapped_ranges.add((vaddr, size))
   def unmap_range(self, vaddr, size): self.mapped_ranges.remove((vaddr, size))
@@ -347,7 +342,7 @@ simd_arrays_per_engine 2
 cu_per_simd_array 8
 simd_per_cu 2
 max_slots_scratch_cu 32
-gfx_target_version {gfx_target_version}
+gfx_target_version 110000
 vendor_id 4098
 device_id 29772
 location_id 34304

test/mockgpu/helpers.py

@@ -16,15 +16,14 @@ def _try_dlopen_gpuocelot():
   return None
 
 class PythonRemu:
-  """Python RDNA3/RDNA4 emulator wrapper that matches the libremu.so interface."""
+  """Python RDNA3 emulator wrapper that matches the libremu.so interface."""
   valid_mem_ranges: set[tuple[int, int]] = set()
   rsrc2: int = 0x19c  # Default: USER_SGPR_COUNT=14, enable X and Y workgroup IDs
   scratch_size: int = 0  # private_segment_fixed_size from kernel descriptor
-  arch: str = "rdna3"  # Architecture: rdna3 or rdna4
 
   def run_asm(self, lib: int, lib_sz: int, gx: int, gy: int, gz: int, lx: int, ly: int, lz: int, args_ptr: int) -> int:
     from extra.assembly.amd.emu import run_asm
-    return run_asm(lib, lib_sz, gx, gy, gz, lx, ly, lz, args_ptr, self.rsrc2, self.scratch_size, self.arch)
+    return run_asm(lib, lib_sz, gx, gy, gz, lx, ly, lz, args_ptr, self.rsrc2, self.scratch_size)
 
 def _try_dlopen_remu():
   # Use Python emulator only if PYTHON_REMU=1

test/models/test_onnx.py

@@ -73,7 +73,7 @@ class TestHuggingFaceOnnxModels(unittest.TestCase):
     onnx_model_path = snapshot_download_with_retry(
       repo_id=repo_id,
       allow_patterns=["*.onnx", "*.onnx_data"],
-      local_dir=DOWNLOADS_DIR / repo_id
+      cache_dir=str(DOWNLOADS_DIR)
     )
     onnx_model_path = onnx_model_path / model_file
     file_size = onnx_model_path.stat().st_size

test/models/test_real_world.py

@@ -12,7 +12,7 @@ from test.helpers import derandomize_model
 
 from examples.gpt2 import Transformer as GPT2Transformer
 from examples.hlb_cifar10 import SpeedyResNet, hyp
-from extra.models.llama import Transformer as LLaMaTransformer
+from examples.llama import Transformer as LLaMaTransformer
 from examples.stable_diffusion import UNetModel, unet_params
 from extra.models.unet import ResBlock
 from extra.models.bert import BertForPretraining

test/null/test_attention.py

@@ -1,39 +0,0 @@
-import unittest
-from tinygrad import Tensor, dtypes, TinyJit, UOp
-from tinygrad.apps.llm import apply_rope as apply_rope_new, precompute_freqs_cis
-
-def apply_rope(x:Tensor, start_pos:int):
-  B, H, T, Hd = x.shape
-  precompute_freqs_cis.cache_clear()
-  freqs_cis = precompute_freqs_cis(Hd, start_pos+T)[start_pos:start_pos+T]
-  return apply_rope_new(x, freqs_cis)
-
-class TestAttention(unittest.TestCase):
-  def test_half_qkv_buffers(self):
-    BS, seqlen, dim = 10, 4, 100
-    q = Tensor.ones(BS, seqlen, dim, dtype=dtypes.half).contiguous().realize()
-    k = Tensor.ones(BS, seqlen, dim, dtype=dtypes.half).contiguous().realize()
-    v = Tensor.ones(BS, seqlen, dim, dtype=dtypes.half).contiguous().realize()
-    attn = q.scaled_dot_product_attention(k, v)
-    sched = attn.schedule()
-    # attention has 4 kernels now
-    self.assertEqual(len(sched), 4)
-
-  def test_apply_rope_jit_prune(self):
-    def rope_fn(x_in, pos): return apply_rope(x_in, pos)
-    rope_noprune = TinyJit(rope_fn)
-    rope_prune = TinyJit(rope_fn, prune=True)
-
-    v_pos = UOp.variable("start_pos", 0, 100)
-    for _ in range(3):
-      rope_noprune(Tensor.randn(1, 2, 4, 8, dtype=dtypes.float32), v_pos.bind(1))
-      rope_prune(Tensor.randn(1, 2, 4, 8, dtype=dtypes.float32), v_pos.bind(1))
-    noprune_size = len(rope_noprune.captured.jit_cache)
-    prune_size = len(rope_prune.captured.jit_cache)
-
-    self.assertGreater(noprune_size, prune_size)
-    self.assertGreaterEqual(noprune_size, 2)
-    self.assertEqual(prune_size, 1)
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_const_folding.py

@@ -1,149 +0,0 @@
-import unittest, itertools, math
-from tinygrad import Tensor, dtypes, Context
-from tinygrad.dtype import DType, ConstType
-from tinygrad.uop.ops import Ops, UOp
-from tinygrad.codegen import full_rewrite_to_sink
-import numpy as np
-
-def _check_ast_count(desired_count:int, t:Tensor):
-  # NOTE: this has side effect because everything can be scheduled only once
-  schedule = t.schedule()
-  asts = [s for s in schedule if s.ast.op is Ops.SINK]
-  assert len(asts) == desired_count, f"{len(asts)} != {desired_count}"
-
-class TestUnaryOpsConstFolding(unittest.TestCase):
-  def test_all_consts_ops(self):
-    _check_ast_count(0, Tensor.ones(4).exp())
-    _check_ast_count(0, Tensor.ones(4).sqrt())
-    _check_ast_count(0, Tensor.ones(4) + Tensor.ones(4))
-    _check_ast_count(0, Tensor.ones(4) / Tensor.ones(4))
-
-  def test_cast(self):
-    _check_ast_count(0, Tensor.ones(4).cast(dtypes.int16))
-    _check_ast_count(0, Tensor.full(4, fill_value=-1).cast(dtypes.uint16))
-
-  def test_neg_folding(self):
-    _check_ast_count(0, Tensor([1, 2, 3]).mul(-1).neg())
-    _check_ast_count(0, Tensor([1, 2, 3]).neg().mul(-1))
-    _check_ast_count(0, Tensor([1, 2, 3]).neg().neg())
-
-  def test_neg_realized_no_fold(self):
-    x = Tensor.randn(32, 32)
-    x = x.clip(0, 1).realize()
-    _check_ast_count(1, x.neg())
-
-class TestBinaryOpsConstFolding(unittest.TestCase):
-  def test_add_literal_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + 0)
-  def test_add_tensor_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) + Tensor.zeros(4))
-  def test_literal_zero_add(self):
-    _check_ast_count(0, 0 + Tensor([1.0, 2, 3, 4]))
-  def test_tensor_zero_add(self):
-    _check_ast_count(0, Tensor.zeros(4) + Tensor([1.0, 2, 3, 4]))
-
-  def test_sub_literal_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - 0)
-  def test_sub_tensor_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) - Tensor.zeros(4))
-
-  def test_mul_literal_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 0)
-  def test_mul_tensor_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.zeros(4))
-  def test_literal_zero_mul(self):
-    _check_ast_count(0, 0 * Tensor([1.0, 2, 3, 4]) * 0)
-  def test_tensor_zero_mul(self):
-    _check_ast_count(0, Tensor.zeros(4) * Tensor([1.0, 2, 3, 4]))
-
-  def test_mul_literal_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * 1)
-  def test_mul_tensor_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) * Tensor.ones(4))
-  def test_literal_one_mul(self):
-    _check_ast_count(0, 1 * Tensor([1.0, 2, 3, 4]))
-  def test_tensor_one_mul(self):
-    _check_ast_count(0, Tensor.ones(4) * Tensor([1.0, 2, 3, 4]))
-
-  def test_bool_tensor_mul_bool(self):
-    _check_ast_count(0, Tensor([True, False]) * True)
-    _check_ast_count(0, Tensor([True, False]) * False)
-  def test_bool_mul_bool_tensor(self):
-    _check_ast_count(0, True * Tensor([True, False]))
-    _check_ast_count(0, False * Tensor([True, False]))
-
-  def test_div_literal_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / 1)
-  def test_div_tensor_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) / Tensor.ones(4))
-
-  def test_idiv_literal_one(self):
-    _check_ast_count(0, Tensor([1, 2, 3, 4]) // 1)
-  def test_idiv_tensor_one(self):
-    _check_ast_count(0, Tensor([1, 2, 3, 4]) // Tensor.ones(4, dtype=dtypes.int32))
-
-  def test_pow_literal_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 0)
-  def test_pow_tensor_zero(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.zeros(4))
-
-  def test_pow_literal_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** 1)
-  def test_pow_tensor_one(self):
-    _check_ast_count(0, Tensor([1.0, 2, 3, 4]) ** Tensor.ones(4))
-  def test_literal_one_pow(self):
-    _check_ast_count(0, 1 ** Tensor([1.0, 2, 3, 4]))
-  def test_tensor_one_pow(self):
-    _check_ast_count(0, Tensor.ones(4) ** Tensor([1.0, 2, 3, 4]))
-
-class TestBitcastConstFolding(unittest.TestCase):
-  def test_scalar_bitcast(self):
-    def t(cases: dict[DType, ConstType]):
-      for (from_dt, from_v), (to_dt, to_v) in itertools.product(cases.items(), cases.items()):
-        if not math.isnan(from_v):
-          r = full_rewrite_to_sink(UOp.const(from_dt, from_v).bitcast(to_dt).sink()).src[0]
-          self.assertEqual(r.op, Ops.CONST, msg:=f"{from_dt} -> {to_dt} ({from_v} -> {to_v})")
-          self.assertEqual(r.dtype, to_dt, msg)
-          np.testing.assert_equal(r.arg, to_v, msg)
-
-    t({dtypes.int8: 0, dtypes.uint8: 0, dtypes.bool: False})
-    t({dtypes.int8: 1, dtypes.uint8: 1, dtypes.bool: True})
-
-    t({dtypes.int8:  -1, dtypes.uint8:  2**8-1})
-    t({dtypes.int16: -1, dtypes.uint16: 2**16-1, dtypes.float16: float('nan')})
-    t({dtypes.int32: -1, dtypes.uint32: 2**32-1, dtypes.float32: float('nan')})
-    t({dtypes.int64: -1, dtypes.uint64: 2**64-1, dtypes.float64: float('nan')})
-
-    t({dtypes.int8:  -2**7,  dtypes.uint8:  2**7})
-    t({dtypes.int16: -2**15, dtypes.uint16: 2**15})
-    t({dtypes.int32: -2**31, dtypes.uint32: 2**31})
-    t({dtypes.int64: -2**63, dtypes.uint64: 2**63})
-
-    t({dtypes.int16: 13496, dtypes.uint16: 13496, dtypes.float16: 0.294921875})
-    t({dtypes.int32: 1050081145, dtypes.uint32: 1050081145, dtypes.float32: 0.29485681653022766})
-    t({dtypes.int64: 4598983288165178391, dtypes.uint64: 4598983288165178391, dtypes.float64: 0.29485681936461233})
-
-  def test_vec_bitcast(self):
-    with Context(SPEC=0):
-      r = full_rewrite_to_sink(UOp.const(dtypes.int32.vec(3), (-1, -2**31, 75)).bitcast(dtypes.uint32.vec(3)).sink()).src[0]
-    self.assertEqual(r.op, Ops.VECTORIZE)
-    self.assertEqual(r.dtype, dtypes.uint32.vec(3))
-    self.assertEqual(tuple(x.arg for x in r.src), (2**32-1, 2**31, 75))
-
-# folds advance indexing into basic indexing
-class TestIndexingConstFolding(unittest.TestCase):
-  def test_scalar_index(self):
-    t = Tensor.arange(16).float().reshape(1,1,4,4).realize()
-    _check_ast_count(1, t[:,:,Tensor(1),:])
-    _check_ast_count(1, t[:,:,Tensor(1)+2,:])
-    _check_ast_count(1, t[:,:,Tensor(1),Tensor(0)])
-
-  def test_const_tensor_index(self):
-    # TODO: these can be 0, implement const tensor folded indexing
-    t = Tensor.arange(16).float().reshape(1,1,4,4).realize()
-    _check_ast_count(1, t[:,:,Tensor.ones(2,1,dtype=dtypes.int),:])
-    _check_ast_count(1, t[:,:,Tensor.ones(1,2,dtype=dtypes.int)+2,:])
-    _check_ast_count(1, t[:,:,Tensor.ones(1,1,dtype=dtypes.int),Tensor.zeros(2,1,2,dtype=dtypes.int)])
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_dtype_spec.py

@@ -1,413 +0,0 @@
-import unittest, math, struct, operator
-from tinygrad.tensor import Tensor, dtypes
-from tinygrad.dtype import DTYPES_DICT, truncate, float_to_fp16, float_to_bf16, _to_np_dtype, least_upper_dtype, least_upper_float
-from tinygrad.device import is_dtype_supported
-
-from tinygrad.helpers import getenv
-from hypothesis import given, settings, strategies as strat
-import numpy as np
-import torch
-
-settings.register_profile("my_profile", max_examples=50, deadline=None, derandomize=getenv("DERANDOMIZE_CI", False))
-settings.load_profile("my_profile")
-
-core_dtypes = list(DTYPES_DICT.values())
-dtype_ints = [dt for dt in core_dtypes if dtypes.is_int(dt) and is_dtype_supported(dt)]
-dtype_floats = [dt for dt in core_dtypes if dtypes.is_float(dt) and is_dtype_supported(dt)]
-
-FP8E4M3_MAX = 448.0
-FP8E5M2_MAX = 57344.0
-
-def u32_to_f32(u): return struct.unpack('f', struct.pack('I', u))[0]
-def f32_to_u32(f): return struct.unpack('I', struct.pack('f', f))[0]
-
-class TestHelpers(unittest.TestCase):
-  signed_ints = (dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64)
-  uints = (dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64)
-  floats = (dtypes.float16, dtypes.float32, dtypes.float64)
-
-  @given(strat.sampled_from(signed_ints+uints), strat.integers(min_value=1, max_value=8))
-  def test_is_int(self, dtype, amt):
-    assert dtypes.is_int(dtype.vec(amt) if amt > 1 else dtype)
-    assert not dtypes.is_float(dtype.vec(amt) if amt > 1 else dtype)
-
-  @given(strat.sampled_from(uints), strat.integers(min_value=1, max_value=8))
-  def test_is_unsigned_uints(self, dtype, amt):
-    assert dtypes.is_unsigned(dtype.vec(amt) if amt > 1 else dtype)
-
-  @given(strat.sampled_from(signed_ints), strat.integers(min_value=1, max_value=8))
-  def test_is_unsigned_signed_ints(self, dtype, amt):
-    assert not dtypes.is_unsigned(dtype.vec(amt) if amt > 1 else dtype)
-
-  @given(strat.sampled_from(floats), strat.integers(min_value=1, max_value=8))
-  def test_is_float(self, dtype, amt):
-    assert dtypes.is_float(dtype.vec(amt) if amt > 1 else dtype)
-    assert not dtypes.is_int(dtype.vec(amt) if amt > 1 else dtype)
-    assert not dtypes.is_unsigned(dtype.vec(amt) if amt > 1 else dtype)
-
-  def test_bf16_is_float(self):
-    assert dtypes.is_float(dtypes.bfloat16)
-
-  def test_fp8s_are_float(self):
-    assert dtypes.is_float(dtypes.fp8e4m3)
-    assert dtypes.is_float(dtypes.fp8e5m2)
-
-  @given(strat.sampled_from([d for d in DTYPES_DICT.values() if dtypes.is_float(d) or dtypes.is_int(d)]), strat.integers(min_value=2, max_value=8))
-  def test_scalar(self, dtype, amt):
-    assert dtype.vec(amt).scalar() == dtype
-
-  def test_from_py(self):
-    assert dtypes.from_py(True) == dtypes.bool
-    assert dtypes.from_py(2) == dtypes.default_int
-    assert dtypes.from_py(3.0) == dtypes.default_float
-    assert dtypes.from_py([]) == dtypes.default_float
-    assert dtypes.from_py(()) == dtypes.default_float
-    assert dtypes.from_py([True]) == dtypes.bool
-    assert dtypes.from_py([True, 2]) == dtypes.default_int
-    assert dtypes.from_py([True, 3.0]) == dtypes.default_float
-    assert dtypes.from_py([2, 3.0]) == dtypes.default_float
-    assert dtypes.from_py([True, 2, 3.0]) == dtypes.default_float
-    with self.assertRaises(RuntimeError): dtypes.from_py(None)
-    with self.assertRaises(RuntimeError): dtypes.from_py([None])
-    with self.assertRaises(RuntimeError): dtypes.from_py({})
-    with self.assertRaises(RuntimeError): dtypes.from_py(set())
-
-  def test_dtype_range(self):
-    for dt in core_dtypes:
-      if dtypes.is_float(dt):
-        np.testing.assert_equal(dtypes.min(dt), -math.inf)
-        np.testing.assert_equal(dtypes.max(dt), math.inf)
-        np.testing.assert_equal(dt.min, -math.inf)
-        np.testing.assert_equal(dt.max, math.inf)
-      elif dtypes.is_int(dt):
-        info = np.iinfo(_to_np_dtype(dt))
-        np.testing.assert_equal(dtypes.min(dt), info.min)
-        np.testing.assert_equal(dtypes.max(dt), info.max)
-        np.testing.assert_equal(dt.min, info.min)
-        np.testing.assert_equal(dt.max, info.max)
-      else:
-        assert dt == dtypes.bool, dt
-        np.testing.assert_equal(dtypes.min(dt), False)
-        np.testing.assert_equal(dtypes.max(dt), True)
-        np.testing.assert_equal(dt.min, False)
-        np.testing.assert_equal(dt.max, True)
-
-  def test_dtype_range_vec(self):
-    for dt in core_dtypes:
-      self.assertEqual(dt.min, dt.vec(4).min)
-      self.assertEqual(dt.max, dt.vec(4).max)
-
-  def test_float_to_fp16(self):
-    self.assertEqual(float_to_fp16(1), 1)
-    self.assertEqual(float_to_fp16(65504), 65504)
-    self.assertEqual(float_to_fp16(65519.999), 65504)
-    self.assertEqual(float_to_fp16(65520), math.inf)
-    self.assertEqual(float_to_fp16(1e-8), 0.0)
-    self.assertEqual(float_to_fp16(-65504), -65504)
-    self.assertEqual(float_to_fp16(-65519.999), -65504)
-    self.assertEqual(float_to_fp16(-65520), -math.inf)
-    self.assertTrue(math.isnan(float_to_fp16(math.nan)))
-
-  def test_float_to_bf16(self):
-    max_bf16 = torch.finfo(torch.bfloat16).max
-    for a in [1, 1.1, 1234, 23456, -777.777, max_bf16, max_bf16 * 1.00001, -max_bf16, -max_bf16 * 1.00001, math.inf, -math.inf]:
-      self.assertEqual(float_to_bf16(a), torch.tensor([a], dtype=torch.bfloat16).item())
-    self.assertTrue(math.isnan(float_to_bf16(math.nan)))
-
-  def test_float_to_bf16_nan(self):
-    patterns = [0x7FC00001, 0xFFC00001, 0x7F800001, 0xFF800001, 0x7FFFFFFF, 0xFFFFFFFF]
-    for u in patterns:
-      x = u32_to_f32(u)
-      y = float_to_bf16(x)
-      t = torch.tensor([x], dtype=torch.bfloat16).item()
-      self.assertTrue(math.isnan(y))
-      self.assertTrue(math.isnan(t))
-
-  def test_float_to_bf16_round(self):
-    uppers = [0x3f800000, 0x41230000, 0xC1460000]
-    for upper in uppers:
-      base = upper & 0xFFFF0000
-      base_f32 = u32_to_f32(base)
-      base_f32_round_up = u32_to_f32(base + 0x00010000)
-
-      x = u32_to_f32(base | 0x00007000)
-      self.assertEqual(float_to_bf16(x), base_f32)
-      self.assertEqual(torch.tensor([x], dtype=torch.bfloat16).item(), base_f32)
-
-      x = u32_to_f32(base | 0x0000C000)
-      self.assertEqual(float_to_bf16(x), base_f32_round_up)
-      self.assertEqual(torch.tensor([x], dtype=torch.bfloat16).item(), base_f32_round_up)
-
-      if ((upper >> 16) & 1) == 0:
-        x = u32_to_f32(base | 0x00008000)
-        self.assertEqual(float_to_bf16(x), base_f32)
-        self.assertEqual(torch.tensor([x], dtype=torch.bfloat16).item(), base_f32)
-      else:
-        x = u32_to_f32(base | 0x00008000)
-        self.assertEqual(float_to_bf16(x), base_f32_round_up)
-        self.assertEqual(torch.tensor([x], dtype=torch.bfloat16).item(), base_f32_round_up)
-
-  def test_float_to_bf16_boundary(self):
-    base = 0x7F7F0000
-    inf_u32 = 0x7F800000
-
-    x = u32_to_f32(base | 0x00007FFF)
-    self.assertEqual(f32_to_u32(float_to_bf16(x)), base)
-    self.assertEqual(f32_to_u32(torch.tensor([x], dtype=torch.bfloat16).item()), base)
-
-    x = u32_to_f32(base | 0x0000C000)
-    self.assertEqual(f32_to_u32(float_to_bf16(x)), inf_u32)
-    self.assertEqual(f32_to_u32(torch.tensor([x], dtype=torch.bfloat16).item()), inf_u32)
-
-    x = u32_to_f32(base | 0x00008000)
-    self.assertEqual(f32_to_u32(float_to_bf16(x)), inf_u32)
-    self.assertEqual(f32_to_u32(torch.tensor([x], dtype=torch.bfloat16).item()), inf_u32)
-
-  @given(strat.floats(width=32, allow_subnormal=True, allow_nan=True, allow_infinity=True))
-  def test_truncate_fp8e4m3(self, x):
-    if math.isnan(x): np.testing.assert_equal(truncate[dtypes.fp8e4m3](x), x)
-    elif math.isinf(x): np.testing.assert_equal(truncate[dtypes.fp8e4m3](x), math.copysign(math.nan, x))
-    elif x > FP8E4M3_MAX: np.testing.assert_equal(truncate[dtypes.fp8e4m3](x), FP8E4M3_MAX)
-    elif x < -FP8E4M3_MAX: np.testing.assert_equal(truncate[dtypes.fp8e4m3](x), -FP8E4M3_MAX)
-    else: np.testing.assert_equal(truncate[dtypes.fp8e4m3](x), torch.tensor(x, dtype=torch.float8_e4m3fn).float().item())
-
-  @given(strat.floats(width=32, allow_subnormal=True, allow_nan=True, allow_infinity=True))
-  def test_truncate_fp8e5m2(self, x):
-    if math.isnan(x): np.testing.assert_equal(truncate[dtypes.fp8e5m2](x), x)
-    elif math.isinf(x): np.testing.assert_equal(truncate[dtypes.fp8e5m2](x), x)
-    elif x > FP8E5M2_MAX: np.testing.assert_equal(truncate[dtypes.fp8e5m2](x), FP8E5M2_MAX)
-    elif x < -FP8E5M2_MAX: np.testing.assert_equal(truncate[dtypes.fp8e5m2](x), -FP8E5M2_MAX)
-    else: np.testing.assert_equal(truncate[dtypes.fp8e5m2](x), torch.tensor(x, dtype=torch.float8_e5m2).float().item())
-
-class TestTypePromotion(unittest.TestCase):
-  @given(strat.sampled_from(core_dtypes))
-  def test_self_promo_to_self(self, dtype):
-    assert least_upper_dtype(dtype) == dtype
-    assert least_upper_dtype(dtype, dtype) == dtype
-    assert least_upper_dtype(dtype, dtype, dtype) == dtype
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes))
-  def test_promo_resulted_higher_than_inputs(self, dtype1, dtype2):
-    result = least_upper_dtype(dtype1, dtype2)
-    assert not (result < dtype1) and not (result < dtype2)
-
-  def test_dtype_promo(self):
-    assert least_upper_dtype(dtypes.bool, dtypes.int8) == dtypes.int8
-    assert least_upper_dtype(dtypes.int8, dtypes.uint8) == dtypes.int16
-    assert least_upper_dtype(dtypes.uint8, dtypes.int16) == dtypes.int16
-    assert least_upper_dtype(dtypes.int16, dtypes.uint16) == dtypes.int32
-    assert least_upper_dtype(dtypes.uint16, dtypes.int32) == dtypes.int32
-    assert least_upper_dtype(dtypes.int32, dtypes.uint32) == dtypes.int64
-    assert least_upper_dtype(dtypes.uint32, dtypes.int64) == dtypes.int64
-    assert least_upper_dtype(dtypes.int64, dtypes.uint64) == dtypes.uint64
-    assert least_upper_dtype(dtypes.float16, dtypes.float32) == dtypes.float32
-    assert least_upper_dtype(dtypes.float32, dtypes.float64) == dtypes.float64
-
-    assert least_upper_dtype(dtypes.bool, dtypes.float32) == dtypes.float32
-    assert least_upper_dtype(dtypes.bool, dtypes.float64) == dtypes.float64
-    assert least_upper_dtype(dtypes.float16, dtypes.int64) == dtypes.float16
-    assert least_upper_dtype(dtypes.float16, dtypes.uint64) == dtypes.float16
-    assert least_upper_dtype(dtypes.fp8e4m3, dtypes.fp8e5m2) == dtypes.half
-    assert least_upper_dtype(dtypes.fp8e4m3, dtypes.bfloat16) == dtypes.bfloat16
-    assert least_upper_dtype(dtypes.fp8e5m2, dtypes.bfloat16) == dtypes.bfloat16
-    assert least_upper_dtype(dtypes.fp8e4m3, dtypes.float16) == dtypes.float16
-    assert least_upper_dtype(dtypes.fp8e5m2, dtypes.float16) == dtypes.float16
-    assert least_upper_dtype(dtypes.fp8e4m3, dtypes.int64) == dtypes.fp8e4m3
-    assert least_upper_dtype(dtypes.fp8e4m3, dtypes.uint64) == dtypes.fp8e4m3
-    assert least_upper_dtype(dtypes.fp8e5m2, dtypes.int64) == dtypes.fp8e5m2
-    assert least_upper_dtype(dtypes.fp8e5m2, dtypes.uint64) == dtypes.fp8e5m2
-
-class TestTypeSpec(unittest.TestCase):
-  def setUp(self):
-    self.old_default_int, self.old_default_float = dtypes.default_int, dtypes.default_float
-  def tearDown(self):
-    dtypes.default_int, dtypes.default_float = self.old_default_int, self.old_default_float
-
-  def test_set_dtype_default(self):
-    for default_int in [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64]:
-      dtypes.default_int = default_int
-      assert dtypes.default_int == default_int
-
-    for default_float in [*dtypes.fp8s, dtypes.float16, dtypes.bfloat16, dtypes.float32, dtypes.float64]:
-      dtypes.default_float = default_float
-      assert dtypes.default_float == default_float
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from([operator.gt, operator.ge, operator.le, operator.lt, operator.eq, operator.ne]))
-  def test_bool_ops(self, dtype, op):
-    assert op(Tensor.ones(4, 4, dtype=dtype), Tensor.ones(4, 4, dtype=dtype)).dtype == dtypes.bool
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(dtype_ints), strat.sampled_from(dtype_floats))
-  def test_functions_return_index(self, dtype, default_int, default_float):
-    dtypes.default_int, dtypes.default_float = default_int, default_float
-    assert Tensor([0, 1], dtype=dtype).argmax().dtype == dtypes.int32
-    assert Tensor([0, 1], dtype=dtype).argmin().dtype == dtypes.int32
-    assert Tensor([0, 1], dtype=dtype).multinomial().dtype == dtypes.int32
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(dtype_ints))
-  def test_tensor_indexing_returns_same_dtype(self, data_dtype, indices_dtype):
-    X_data =  Tensor.ones(60000, 1, 28, 28, dtype=data_dtype)
-    indices =  Tensor.randint(512, high=X_data.shape[0]).cast(indices_dtype)
-    assert X_data[indices].dtype == X_data.dtype
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(dtype_ints))
-  def test_gather_returns_same_dtype(self, data_dtype, indices_dtype):
-    X_data = Tensor([[1, 0], [0, 1]], dtype=data_dtype)
-    indices = Tensor([[0, 0], [1, 0]], dtype=indices_dtype)
-    assert X_data.gather(0, indices).dtype == X_data.dtype
-    assert X_data.gather(1, indices).dtype == X_data.dtype
-
-  @given(strat.sampled_from(dtype_floats), strat.sampled_from(dtype_floats))
-  def test_attention_returns_same_dtype(self, data_dtype, default_float):
-    dtypes.default_float = default_float
-    query = Tensor.rand(32, 8, 128, 64, dtype=data_dtype)
-    key = Tensor.rand(32, 8, 128, 64, dtype=data_dtype)
-    value = Tensor.rand(32, 8, 128, 64, dtype=data_dtype)
-    mask = (Tensor.rand(32, 8, 128, 128) < 0.5)
-    assert query.scaled_dot_product_attention(key, value, is_causal=True).dtype == data_dtype
-    assert query.scaled_dot_product_attention(key, value, is_causal=True, dropout_p=0.3).dtype == data_dtype
-    assert query.scaled_dot_product_attention(key, value, is_causal=False).dtype == data_dtype
-    assert query.scaled_dot_product_attention(key, value, attn_mask=mask).dtype == data_dtype
-
-class TestAutoCastType(unittest.TestCase):
-  def setUp(self):
-    self.old_default_int, self.old_default_float = dtypes.default_int, dtypes.default_float
-  def tearDown(self):
-    dtypes.default_int, dtypes.default_float = self.old_default_int, self.old_default_float
-
-  @given(strat.sampled_from(dtype_floats), strat.sampled_from(dtype_floats))
-  def test_least_upper_float_input_is_float(self, input_dtype, default_float):
-    dtypes.default_float = default_float
-    self.assertEqual(least_upper_float(input_dtype), input_dtype)
-
-  @given(strat.sampled_from(dtype_ints), strat.sampled_from(dtype_floats))
-  def test_least_upper_float_input_is_int(self, input_dtype, default_float):
-    dtypes.default_float = default_float
-    self.assertEqual(least_upper_float(input_dtype), default_float)
-
-  @given(strat.sampled_from(core_dtypes))
-  def test_broadcast_scalar(self, dt):
-    assert (Tensor.ones(4, 4, dtype=dt) + 2.3).dtype == (dt if dtypes.is_float(dt) else dtypes.default_float)
-    assert (Tensor.ones(4, 4, dtype=dt) + 2).dtype == (dt if dtypes.is_float(dt) or dtypes.is_int(dt) else dtypes.default_int)
-    assert (Tensor.ones(4, 4, dtype=dt) + True).dtype == dt
-
-  @given(strat.sampled_from(dtype_floats))
-  def test_int_div_int(self, default_float):
-    dtypes.default_float = default_float
-    self.assertEqual(Tensor([1]).div(Tensor([2])).dtype, default_float)
-
-  def test_sum(self):
-    assert (Tensor([0, 1], dtype=dtypes.bool)).sum().dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int8)).sum().dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int16)).sum().dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int32)).sum().dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int64)).sum().dtype == dtypes.int64
-    assert (Tensor([0, 1], dtype=dtypes.uint8)).sum().dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint16)).sum().dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint32)).sum().dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint64)).sum().dtype == dtypes.uint64
-    assert (Tensor([0, 1], dtype=dtypes.fp8e4m3)).sum().dtype == dtypes.fp8e4m3
-    assert (Tensor([0, 1], dtype=dtypes.fp8e5m2)).sum().dtype == dtypes.fp8e5m2
-    assert (Tensor([0, 1], dtype=dtypes.float16)).sum().dtype == dtypes.float16
-    assert (Tensor([0, 1], dtype=dtypes.bfloat16)).sum().dtype == dtypes.bfloat16
-    assert (Tensor([0, 1], dtype=dtypes.float32)).sum().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.float64)).sum().dtype == dtypes.float64
-
-  def test_mean(self):
-    assert (Tensor([0, 1], dtype=dtypes.bool)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.int8)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.int16)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.int32)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.int64)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.uint8)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.uint16)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.uint32)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.uint64)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.fp8e4m3)).mean().dtype == dtypes.fp8e4m3
-    assert (Tensor([0, 1], dtype=dtypes.fp8e5m2)).mean().dtype == dtypes.fp8e5m2
-    assert (Tensor([0, 1], dtype=dtypes.float16)).mean().dtype == dtypes.float16
-    assert (Tensor([0, 1], dtype=dtypes.bfloat16)).mean().dtype == dtypes.bfloat16
-    assert (Tensor([0, 1], dtype=dtypes.float32)).mean().dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.float64)).mean().dtype == dtypes.float64
-
-  def test_cumsum(self):
-    assert (Tensor([0, 1], dtype=dtypes.bool)).cumsum(0).dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int8)).cumsum(0).dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int16)).cumsum(0).dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int32)).cumsum(0).dtype == dtypes.int32
-    assert (Tensor([0, 1], dtype=dtypes.int64)).cumsum(0).dtype == dtypes.int64
-    assert (Tensor([0, 1], dtype=dtypes.uint8)).cumsum(0).dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint16)).cumsum(0).dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint32)).cumsum(0).dtype == dtypes.uint32
-    assert (Tensor([0, 1], dtype=dtypes.uint64)).cumsum(0).dtype == dtypes.uint64
-    assert (Tensor([0, 1], dtype=dtypes.fp8e4m3)).cumsum(0).dtype == dtypes.fp8e4m3
-    assert (Tensor([0, 1], dtype=dtypes.fp8e5m2)).cumsum(0).dtype == dtypes.fp8e5m2
-    assert (Tensor([0, 1], dtype=dtypes.float16)).cumsum(0).dtype == dtypes.float16
-    assert (Tensor([0, 1], dtype=dtypes.bfloat16)).cumsum(0).dtype == dtypes.bfloat16
-    assert (Tensor([0, 1], dtype=dtypes.float32)).cumsum(0).dtype == dtypes.float32
-    assert (Tensor([0, 1], dtype=dtypes.float64)).cumsum(0).dtype == dtypes.float64
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes))
-  def test_matmul(self, dt1, dt2, acc_dt):
-    t1 = Tensor([0, 1], dtype=dt1)
-    t2 = Tensor([0, 1], dtype=dt2)
-    self.assertEqual(t1.matmul(t2).dtype, least_upper_dtype(t1.dtype, t2.dtype))
-    self.assertEqual(t1.matmul(t2, dtype=acc_dt).dtype, acc_dt)
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes))
-  def test_linear(self, dt1, dt2, dt3, acc_dt):
-    x = Tensor([0, 1], dtype=dt1)
-    w = Tensor([0, 1], dtype=dt2)
-    b = Tensor([0, 1], dtype=dt3)
-    self.assertEqual(x.linear(w).dtype, least_upper_dtype(x.dtype, w.dtype))
-    self.assertEqual(x.linear(w, b).dtype, least_upper_dtype(least_upper_dtype(x.dtype, w.dtype), b.dtype))
-    self.assertEqual(x.linear(w, dtype=acc_dt).dtype, acc_dt)
-    self.assertEqual(x.linear(w, b, dtype=acc_dt).dtype, acc_dt)
-
-  @staticmethod
-  def check_where_alternate_input_other(input_, other, data_type):
-    assert (Tensor([True, False]).where(input_, other)).dtype == data_type
-    assert (Tensor([True, False]).where(other, input_)).dtype == data_type
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes))
-  def test_where_no_scalar(self, dt1, dt2):
-    self.check_where_alternate_input_other(Tensor(2, dtype=dt1), Tensor(3, dtype=dt2), least_upper_dtype(dt1, dt2))
-
-  @given(strat.sampled_from(core_dtypes))
-  def test_where_one_scalar(self, dt):
-    t = Tensor(2, dtype=dt)
-    self.check_where_alternate_input_other(t, 3.2, (dt if dtypes.is_float(dt) else dtypes.default_float))
-    self.check_where_alternate_input_other(t, 3, (dt if dtypes.is_float(dt) or dtypes.is_int(dt) else dtypes.default_int))
-    self.check_where_alternate_input_other(t, True, dt)
-
-  def test_where_two_scalars(self):
-    self.check_where_alternate_input_other(3.1, 3.2, dtypes.default_float)
-    self.check_where_alternate_input_other(3.1, 3, dtypes.default_float)
-    self.check_where_alternate_input_other(3.1, True, dtypes.default_float)
-    self.check_where_alternate_input_other(3, 2, dtypes.default_int)
-    self.check_where_alternate_input_other(3, True, dtypes.default_int)
-    self.check_where_alternate_input_other(False, True, dtypes.bool)
-
-  @given(strat.sampled_from(core_dtypes), strat.sampled_from(core_dtypes))
-  def test_maximum(self, dt1, dt2):
-    assert Tensor([0, 1, 2], dtype=dt1).maximum(Tensor([2, 0, 5], dtype=dt2)).dtype == least_upper_dtype(dt1, dt2)
-
-  @given(strat.sampled_from(core_dtypes))
-  def test_maximum_const(self, dt):
-    assert Tensor([1, 2], dtype=dt).maximum(3.1).dtype == (dt if dtypes.is_float(dt) else dtypes.default_float)
-    assert Tensor([1, 2], dtype=dt).maximum(3).dtype == (dt if dtypes.is_float(dt) or dtypes.is_int(dt) else dtypes.default_int)
-    assert Tensor([1, 2], dtype=dt).maximum(True).dtype == dt
-
-  def test_div(self):
-    assert (Tensor([1, 2], dtype=dtypes.int32) / Tensor([2, 2], dtype=dtypes.int32)).dtype == dtypes.default_float
-    assert (Tensor([1, 2], dtype=dtypes.int16) / Tensor([2, 2], dtype=dtypes.int32)).dtype == dtypes.default_float
-    assert (Tensor([1, 2], dtype=dtypes.float32) / Tensor([2, 2], dtype=dtypes.float16)).dtype == dtypes.float32
-    assert (Tensor([1, 2], dtype=dtypes.int32) / Tensor([2, 2], dtype=dtypes.float16)).dtype == dtypes.float16
-
-  def test_div_const(self):
-    assert (Tensor([1, 2], dtype=dtypes.int32) / 2).dtype == dtypes.default_float
-    assert (Tensor([1, 2], dtype=dtypes.int32) / 2.0).dtype == dtypes.default_float
-    assert (Tensor([1, 2], dtype=dtypes.float16) / 2).dtype == dtypes.float16
-    assert (Tensor([1, 2], dtype=dtypes.float16) / 2.0).dtype == dtypes.float16
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_gradient.py

@@ -1,83 +0,0 @@
-from typing import Callable
-import unittest, math
-import torch
-from tinygrad import Tensor
-from tinygrad.dtype import dtypes
-from tinygrad.uop.ops import UOp
-from tinygrad.gradient import compute_gradient
-
-class TestGradient(unittest.TestCase):
-  def _cmp_nan_okay(self, x, y):
-    if math.isnan(x) and math.isnan(y): return
-    self.assertAlmostEqual(x, y, places=5)
-
-  def _test_one_input_function(self, f:Callable, jf:Callable|None=None):
-    if jf is None: jf = f
-    x = UOp.variable('x', -math.inf, math.inf, dtype=dtypes.float)
-    gx = compute_gradient(f(x), UOp.const(dtypes.float, 1.0), set([x]))[x]
-
-    for val in [-5., -2.0, 0.0, 2.0, 5.]:
-      tg_out = gx.substitute({x: x.const_like(val)}).ssimplify()
-      tx = torch.tensor([val], dtype=torch.float, requires_grad=True)
-      torch_out = torch.autograd.grad(jf(tx), tx)[0].item()
-      self._cmp_nan_okay(tg_out, torch_out)
-
-  def _test_two_input_function(self, f:Callable, jf:Callable|None=None):
-    if jf is None: jf = f
-    x = UOp.variable('x', -math.inf, math.inf, dtype=dtypes.float)
-    y = UOp.variable('y', -math.inf, math.inf, dtype=dtypes.float)
-    grads = compute_gradient(f(x, y), UOp.const(dtypes.float, 1.0), set([x, y]))
-    gx, gy = grads[x], grads[y]
-
-    for valx in [-5., -2.0, 0.0, 2.0, 5.]:
-      for valy in [-5., -2.0, 0.0, 2.0, 5.]:
-        # Substitute the values into the gradient expressions
-        substitutions = {x: x.const_like(valx), y: y.const_like(valy)}
-        tg_out_x = gx.substitute(substitutions).ssimplify()
-        tg_out_y = gy.substitute(substitutions).ssimplify()
-
-        tx = torch.tensor([valx], dtype=torch.float, requires_grad=True)
-        ty = torch.tensor([valy], dtype=torch.float, requires_grad=True)
-        torch_grad = torch.autograd.grad(jf(tx, ty), [tx, ty])
-        torch_out_x, torch_out_y = [x.item() for x in torch_grad]
-
-        self._cmp_nan_okay(tg_out_x, torch_out_x)
-        self._cmp_nan_okay(tg_out_y, torch_out_y)
-
-  # unary ops unit
-  def test_recip(self): self._test_one_input_function(lambda x: 1.0/x)
-  def test_sin(self): self._test_one_input_function(lambda x: x.sin())
-  def test_sqrt(self): self._test_one_input_function(lambda x: x.sqrt())
-  def test_log2(self): self._test_one_input_function(lambda x: x.log2())
-  def test_exp2(self): self._test_one_input_function(lambda x: x.exp2())
-
-  # binary ops unit
-  def test_add(self): self._test_two_input_function(lambda x,y: x+y)
-  def test_mul(self): self._test_two_input_function(lambda x,y: x*y)
-
-  # chain rule
-  def test_chain(self): self._test_one_input_function(lambda x: x.sin().sqrt())
-  def test_chain_binop(self): self._test_two_input_function(lambda x,y: (x*y)+x*y)
-  def test_big_add_sin(self): self._test_two_input_function(lambda x,y: x.sin()+3.0/y)
-  def test_big_chain(self): self._test_two_input_function(lambda x,y: (1.0/x*y)+x*y)
-  def test_where(self): self._test_two_input_function(lambda x,y: (x<y).where(x,y), lambda x,y: torch.where(x<y,x,y))
-
-class TestRealizeMeansRealize(unittest.TestCase):
-  def test_randn_realizes(self):
-    x = Tensor.randn(2, 3, 64, 64, requires_grad=True).realize()
-    assert x.uop is not x.uop.base
-    assert x.uop.is_realized
-
-  def test_uniform_realizes(self):
-    x = Tensor.uniform(16, 3, 3, 3, requires_grad=True).realize()
-    print(x.uop)
-    assert x.uop is not x.uop.base
-    assert x.uop.is_realized
-
-  def test_uniform_gradient(self):
-    x = Tensor.uniform(16, 3, 3, 3, requires_grad=True).realize()
-    y = x * 2
-    y.sum().gradient(x)[0].realize()
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_helpers.py

@@ -1,469 +0,0 @@
-import ctypes, gzip, unittest, timeit, pickle
-from tinygrad import Variable
-from tinygrad.helpers import Context, ContextVar, argfix, colored, word_wrap, is_numpy_ndarray, mv_address, get_contraction, count, all_same
-from tinygrad.helpers import merge_dicts, strip_parens, prod, round_up, fetch, fully_flatten, from_mv, to_mv, polyN, time_to_str, cdiv, cmod, getbits
-from tinygrad.helpers import ceildiv
-from tinygrad.tensor import Tensor, get_shape
-import numpy as np
-
-VARIABLE = ContextVar("VARIABLE", 0)
-
-class TestContextVars(unittest.TestCase):
-  # Ensuring that the test does not modify variables outside the tests.
-  ctx = Context()
-  def setUp(self): TestContextVars.ctx.__enter__()
-  def tearDown(self): TestContextVars.ctx.__exit__()
-
-  def test_initial_value_is_set(self):
-    _TMP = ContextVar("_TMP", 5)
-    self.assertEqual(_TMP.value, 5)
-
-  def test_cannot_recreate(self):
-    _TMP2 = ContextVar("_TMP2", 1)
-    with self.assertRaises(RuntimeError):
-      _TMP2 = ContextVar("_TMP2", 2)
-
-  def test_new_var_inside_context(self):
-    with Context(VARIABLE=1):
-      _TMP3 = ContextVar("_TMP3", 1)
-    with self.assertRaises(RuntimeError):
-      _TMP3 = ContextVar("_TMP3", 2)
-
-  def test_value_across_modules(self):
-    # Mocking module import by invoking the code but not in our globals().
-    exec('from tinygrad.helpers import ContextVar;C = ContextVar("C", 13)', {}) # pylint:disable=exec-used
-    # It should not matter that the first creation was in another module.
-    with self.assertRaises(RuntimeError):
-      _C = ContextVar("C", 0)
-
-  def test_assignment_across_modules(self):
-    B = ContextVar("B", 1)
-    # local assignment
-    B.value = 2
-    self.assertEqual(B.value, 2)
-    with self.assertRaises(RuntimeError):
-      # Assignment in another module.
-      exec('from tinygrad.helpers import ContextVar;B = ContextVar("B", 0);B.value = 3;', {}) # pylint:disable=exec-used
-
-  def test_context_assignment(self):
-    with Context(VARIABLE=1):
-      self.assertEqual(VARIABLE.value, 1)
-    self.assertEqual(VARIABLE.value, 0)
-
-  def test_unknown_param_to_context(self):
-    with self.assertRaises(KeyError):
-      with Context(SOMETHING_ELSE=1):
-        pass
-
-  def test_nested_context(self):
-    with Context(VARIABLE=1):
-      with Context(VARIABLE=2):
-        MORE = ContextVar("MORE", 2)
-        with Context(VARIABLE=3, MORE=3):
-          self.assertEqual(VARIABLE.value, 3)
-          self.assertEqual(MORE.value, 3)
-        self.assertEqual(VARIABLE.value, 2)
-        self.assertEqual(MORE.value, 2)
-      self.assertEqual(VARIABLE.value, 1)
-      self.assertEqual(MORE.value, 2)  # TODO: should this raise?
-    self.assertEqual(VARIABLE.value, 0)
-
-  def test_decorator(self):
-    @Context(VARIABLE=1, DEBUG=4)
-    def test():
-      self.assertEqual(VARIABLE.value, 1)
-
-    self.assertEqual(VARIABLE.value, 0)
-    test()
-    self.assertEqual(VARIABLE.value, 0)
-
-  def test_context_exit_reverts_updated_values(self):
-    D = ContextVar("D", 1)
-    D.value = 2
-    with Context(D=3):
-      ...
-    assert D.value == 2, f"Expected D to be 2, but was {D.value}. Indicates that Context.__exit__ did not restore to the correct value."
-
-class TestAllSame(unittest.TestCase):
-  def test_empty(self): self.assertTrue(all_same([]))
-  def test_single(self): self.assertTrue(all_same([1]))
-  def test_same(self): self.assertTrue(all_same([1, 1, 1]))
-  def test_different(self): self.assertFalse(all_same([1, 2, 1]))
-
-class TestMergeDicts(unittest.TestCase):
-  def test_merge_dicts(self):
-    a = {"a": 1, "b": 2}
-    b = {"a": 1, "c": 3}
-    c = {}
-    d = {"a": 2, "b": 2}
-    assert merge_dicts([a, b]) == {"a": 1, "b": 2, "c": 3}
-    assert merge_dicts([a, c]) == a
-    assert merge_dicts([a, b, c]) == {"a": 1, "b": 2, "c": 3}
-    with self.assertRaises(RuntimeError):
-      merge_dicts([a, d])
-
-class TestStripParens(unittest.TestCase):
-  def test_simple(self): self.assertEqual("1+2", strip_parens("(1+2)"))
-  def test_nested(self): self.assertEqual("1+(2+3)", strip_parens("(1+(2+3))"))
-  def test_casted_no_strip(self): self.assertEqual("(int)(1+2)", strip_parens("(int)(1+2)"))
-  def test_unmatched_parens(self): self.assertEqual("((c35+c39>>23&255)+-127).cast(dtypes.float)",
-    strip_parens("((c35+c39>>23&255)+-127).cast(dtypes.float)"))
-  def test_single_paren_left(self): self.assertEqual("(abc", strip_parens("(abc"))
-  def test_single_paren_right(self): self.assertEqual("abc)", strip_parens("abc)"))
-  def test_parens_at_different_depths(self): self.assertEqual("(a+(b))*(c)", strip_parens("(a+(b))*(c)"))
-
-class TestProd(unittest.TestCase):
-  def test_empty(self): self.assertEqual(1, prod(tuple()))
-  def test_ints(self): self.assertEqual(30, prod((2, 3, 5)))
-  def test_variable(self): self.assertEqual("(a*12)", prod((Variable("a", 1, 5), 3, 4)).render())
-  def test_variable_order(self): self.assertEqual("(a*12)", prod((3, 4, Variable("a", 1, 5))).render())
-
-class TestRoundUp(unittest.TestCase):
-  def test_round_up(self):
-    self.assertEqual(round_up(-3,4), 0)
-    self.assertEqual(round_up(-4,4), -4)
-    self.assertEqual(round_up(6,4), 8)
-    self.assertEqual(round_up(8,4), 8)
-    self.assertEqual(round_up(232, 24984), 24984)
-    self.assertEqual(round_up(24984, 232), 25056)
-
-class TestCeilDiv(unittest.TestCase):
-  def test_int(self):
-    self.assertEqual(ceildiv(10, 3), 4)
-    self.assertEqual(ceildiv(9, 3), 3)
-    self.assertEqual(ceildiv(0, 5), 0)
-    self.assertEqual(ceildiv(1, 5), 1)
-  def test_symbolic(self):
-    # tests that ceildiv with UOp uses (num + amt - 1) // amt formula for non-negative num
-    v = Variable('v', 0, 100)
-    result = ceildiv(v, 6)
-    self.assertEqual(result.render(), "((v+5)//6)")
-  def test_symbolic_negative_offset(self):
-    # tests ceildiv(v-5, 6) which is used in conv2d output shape
-    # old implementation incorrectly simplified -(x//-y) to ((v+1)//6-1) for v-5
-    # new implementation uses (v-5+5)//6 = v//6 which is correct
-    v = Variable('v', 11, 100)
-    result = ceildiv(v - 5, 6)
-    self.assertEqual(result.render(), "(v//6)")
-
-class TestCount(unittest.TestCase):
-  def test_count_basic(self):
-    c = count(3)
-    self.assertEqual(next(c), 3)
-    self.assertEqual(next(c), 4)
-
-  def test_count_step_pickle(self):
-    c = count(1, 2)
-    self.assertEqual(next(c), 1)
-    c2 = pickle.loads(pickle.dumps(c))
-    self.assertEqual(next(c2), 3)
-
-@unittest.skip("no fetch tests because they need internet")
-class TestFetch(unittest.TestCase):
-  def test_fetch_bad_http(self):
-    self.assertRaises(Exception, fetch, 'http://www.google.com/404', allow_caching=False)
-
-  def test_fetch_small(self):
-    assert (len(fetch('https://google.com', allow_caching=False).read_bytes())>0)
-
-  def test_fetch_img(self):
-    from PIL import Image
-    img = fetch("https://avatars.githubusercontent.com/u/132956020", allow_caching=False)
-    with Image.open(img) as pimg:
-      assert pimg.size == (77, 77), pimg.size
-
-  def test_fetch_subdir(self):
-    from PIL import Image
-    img = fetch("https://avatars.githubusercontent.com/u/132956020", allow_caching=False, subdir="images")
-    with Image.open(img) as pimg:
-      assert pimg.size == (77, 77), pimg.size
-    assert img.parent.name == "images"
-
-  def test_fetch_gunzip_valid(self):
-    # compare fetch(gunzip=True) to fetch(gunzip=False) plus decompressing afterwards
-    gzip_url: str = 'https://ftp.gnu.org/gnu/gzip/gzip-1.13.tar.gz'
-    fp_gz = fetch(gzip_url, gunzip=True)
-    fp_no_gz = fetch(gzip_url, gunzip=False)
-    with open(fp_gz, 'rb') as f: content_gz = f.read()
-    with open(fp_no_gz, 'rb') as f: content_no_gz = gzip.decompress(f.read())
-    assert fp_gz.stat().st_size > fp_no_gz.stat().st_size
-    assert isinstance(content_gz, bytes) and isinstance(content_no_gz, bytes)
-    assert len(content_gz) == len(content_no_gz)
-    assert content_gz == content_no_gz
-
-  def test_fetch_gunzip_invalid(self):
-    # given a non-gzipped file, fetch(gunzip=True) fails
-    no_gzip_url: str = 'https://ftp.gnu.org/gnu/gzip/gzip-1.13.zip'
-    with self.assertRaises(gzip.BadGzipFile):
-      fetch(no_gzip_url, gunzip=True)
-
-  def test_fetch_user_agent(self):
-    fetch("https://csrc.nist.gov/CSRC/media/Projects/lightweight-cryptography/documents/finalist-round/updated-submissions/sparkle.zip",
-          allow_caching=False)
-
-  def test_fetch_half_and_full_file(self):
-    x = fetch("https://csrc.nist.gov/CSRC/media/Projects/lightweight-cryptography/documents/finalist-round/updated-submissions/sparkle.zip",
-          headers={"Range": "bytes=0-10"}).read_bytes()
-    assert len(x) == 11, f"{len(x) != 11}"
-    x = fetch("https://csrc.nist.gov/CSRC/media/Projects/lightweight-cryptography/documents/finalist-round/updated-submissions/sparkle.zip",
-          headers={"Range": "bytes=0-100"}).read_bytes()
-    assert len(x) == 101, f"{len(x) != 101}"
-
-class TestFullyFlatten(unittest.TestCase):
-  def test_fully_flatten(self):
-    self.assertEqual(fully_flatten([[1, 3], [1, 2]]), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten(((1, 3), (1, 2))), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten([[[1], [3]], [[1], [2]]]), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten([[[[1], 2], 3], 4]), [1, 2, 3, 4])
-    self.assertEqual(fully_flatten([[1, 2, [3, 4]], [5, 6], 7]), [1, 2, 3, 4, 5, 6, 7])
-    self.assertEqual(fully_flatten([[1, "ab"], [True, None], [3.14, [5, "b"]]]), [1, "ab", True, None, 3.14, 5, "b"])
-
-  def test_fully_flatten_numpy(self):
-    self.assertEqual(fully_flatten([np.array([])]), [])
-    self.assertEqual(fully_flatten([np.array(3)]), [3])
-    self.assertEqual(fully_flatten([np.array([3])]), [3])
-    self.assertEqual(fully_flatten([np.array([[3]])]), [3])
-    self.assertEqual(fully_flatten([np.array([1, 3]), np.array([1, 2])]), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten((np.array([1, 3]), np.array([1, 2]))), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten([np.array([[1], [3]]), np.array([[1], [2]])]), [1, 3, 1, 2])
-    self.assertEqual(fully_flatten([[1, "ab"], [True, None], np.array([[3.14], [6.28]])]), [1, "ab", True, None, 3.14, 6.28])
-
-class TestMemoryview(unittest.TestCase):
-  def test_from_mv_to_mv(self):
-    base = memoryview(bytearray(b"\x11\x22\x33"*40))
-    ct = from_mv(base)
-    mv = to_mv(ctypes.addressof(ct), len(base))
-    mv[0] = 2
-    assert base[0] == 2
-
-  @unittest.skip("allocates tons of memory")
-  def test_to_mv(self):
-    sizes = [
-      (16, "16 B"),
-      (64, "64 B"),
-      (256, "256 B"),
-      (1024, "1 KB"),
-      (4 * 1024, "4 KB"),
-      (16 * 1024, "16 KB"),
-      (64 * 1024, "64 KB"),
-      (256 * 1024, "256 KB"),
-      (1 * 1024 * 1024, "1 MB"),
-      (10 * 1024 * 1024, "10 MB"),
-      (200 * 1024 * 1024, "200 MB"),
-    ]
-
-    for sz, label in sizes:
-      buf = np.random.randint(0, 256, sz, dtype=np.uint8)
-      ptr = buf.ctypes.data
-
-      iters = 100_000
-      t_us = timeit.timeit(lambda: to_mv(ptr, sz), number=iters) * 1e6 / iters
-      print(f"Size {label:>9} | Time: {t_us:8.3f} µs")
-
-  def test_speed_from_mv_vs_mv_address(self):
-    x = memoryview(bytearray(1))
-
-    iters = 100000
-    fmv_us = timeit.timeit(lambda: from_mv(x), number=iters) * 1e6 / iters
-    mva_us = timeit.timeit(lambda: mv_address(x), number=iters) * 1e6 / iters
-    print(f"from_mv vs mv_address: {fmv_us:8.3f} µs vs {mva_us:8.3f} µs")
-
-class TestGetContraction(unittest.TestCase):
-  def test_contraction(self):
-    r = get_contraction((1,2,3,4), (2,3,4))
-    self.assertEqual(r, [[0, 1], [2], [3]])
-
-    r = get_contraction((2,1,3,4), (2,3,4))
-    self.assertEqual(r, [[0], [1, 2], [3]])
-
-    r = get_contraction((1,2,3,1,4), (1,2,3,4))
-    self.assertEqual(r, [[], [0, 1], [2], [3, 4]])
-
-    r = get_contraction((1,2,3,1,4,1,1), (2,3,4))
-    self.assertEqual(r, [[0, 1], [2], [3, 4, 5, 6]])
-
-    r = get_contraction((1,2,3,4), (1,2,3*4))
-    self.assertEqual(r, [[], [0, 1], [2, 3]])
-
-    r = get_contraction((1,2,3,4), (2,1,3,4))
-    self.assertEqual(r, [[0, 1], [], [2], [3]])
-
-    r = get_contraction((1,2,3,4), (1,1,2*3*4,1))
-    self.assertEqual(r, [[], [], [0,1,2,3], []])
-
-    r = get_contraction((2,1,3,4), (1,2,3,4))
-    self.assertEqual(r, [[], [0], [1, 2], [3]])
-
-    r = get_contraction((1,2,3,4), (2*3*4,1,1,1))
-    self.assertEqual(r, [[0, 1, 2, 3], [], [], []])
-
-    r = get_contraction((4,4,4,4), (16,1,16))
-    self.assertEqual(r, [[0, 1], [], [2, 3]])
-
-    r = get_contraction((1,2,3,4,1,1,1), (2,3,4))
-    self.assertEqual(r, [[0, 1], [2], [3, 4, 5, 6]])
-
-    r = get_contraction((1,2,3,4), (1,2,3,4,1))
-    self.assertEqual(r, [[], [0, 1], [2], [3], []])
-
-    r = get_contraction((14,1,384,14,1,1,1,1), (1,14,384,14))
-    self.assertEqual(r, [[], [0], [1,2], [3,4,5,6,7]])
-
-    r = get_contraction((14,1,384,1,14,1,1,1,1), (1,14,384,14))
-    self.assertEqual(r, [[], [0], [1,2], [3,4,5,6,7,8]])
-
-    r = get_contraction((512, 512), (1, 1, 512, 1, 1, 1, 1, 512))
-    self.assertEqual(r, [[], [], [0], [], [], [], [], [1]])
-
-    r = get_contraction((1,2,3,4), (1,2,6,2))
-    self.assertEqual(r, None)
-
-  def test_contraction_ones(self):
-    r = get_contraction((1,), (1,1,1))
-    self.assertEqual(r, [[], [], [0]])
-
-    r = get_contraction((1,1), (1,1,1))
-    self.assertEqual(r, [[], [], [0, 1]])
-
-    r = get_contraction((1,1,1,1), (1,))
-    self.assertEqual(r, [[0,1,2,3]])
-
-    r = get_contraction((1,1,1,1), (1,1))
-    self.assertEqual(r, [[], [0,1,2,3]])
-
-    r = get_contraction((1,1,1,1), (1,1,1))
-    self.assertEqual(r, [[], [], [0,1,2,3]])
-
-    r = get_contraction((1,1,1,1), (1,1,1,1))
-    self.assertEqual(r, [[], [], [], [0,1,2,3]])
-
-class TestGetShape(unittest.TestCase):
-  def test_get_shape(self):
-    assert get_shape(2) == ()
-    assert get_shape([]) == (0,)
-    assert get_shape([[]]) == (1, 0)
-    assert get_shape([[1, 2]]) == (1, 2)
-    assert get_shape([[1, 2], (3, 4)]) == (2, 2)
-
-  def test_inhomogeneous_shape(self):
-    with self.assertRaises(ValueError): get_shape([[], [1]])
-    with self.assertRaises(ValueError): get_shape([[1, [2]], [1]])
-
-class TestPolyN(unittest.TestCase):
-  def test_float(self):
-    np.testing.assert_allclose(polyN(1.0, [1.0, -2.0, 1.0]), 0.0)
-    np.testing.assert_allclose(polyN(2.0, [1.0, -2.0, 1.0]), 1.0)
-    np.testing.assert_allclose(polyN(3.0, [1.0, -2.0, 1.0]), 4.0)
-    np.testing.assert_allclose(polyN(4.0, [1.0, -2.0, 1.0]), 9.0)
-
-  def test_uop(self):
-    from tinygrad.dtype import dtypes
-    from tinygrad.uop.ops import UOp
-    from test.helpers import eval_uop
-    np.testing.assert_allclose(eval_uop(polyN(UOp.const(dtypes.float, 1.0), [1.0, -2.0, 1.0])), 0.0)
-    np.testing.assert_allclose(eval_uop(polyN(UOp.const(dtypes.float, 2.0), [1.0, -2.0, 1.0])), 1.0)
-    np.testing.assert_allclose(eval_uop(polyN(UOp.const(dtypes.float, 3.0), [1.0, -2.0, 1.0])), 4.0)
-    np.testing.assert_allclose(eval_uop(polyN(UOp.const(dtypes.float, 4.0), [1.0, -2.0, 1.0])), 9.0)
-
-class TestTimeToStr(unittest.TestCase):
-  def test_seconds(self):           self.assertEqual("   10.01s ", time_to_str(10.01))
-  def test_boundary_sec_ms(self):   self.assertEqual("10000.00ms", time_to_str(10))
-  def test_milliseconds(self):      self.assertEqual("  500.00ms", time_to_str(0.5))
-  def test_boundary_ms_us(self):    self.assertEqual("10000.00us", time_to_str(0.01))
-  def test_microseconds(self):      self.assertEqual("  100.00us", time_to_str(0.0001))
-  def test_zero(self):              self.assertEqual("    0.00us", time_to_str(0))
-  def test_width_formatting(self):  self.assertEqual(" 10.01s ", time_to_str(10.01, w=6))
-
-class TestCStyleDivMod(unittest.TestCase):
-  def test_div_pos(self):
-    self.assertEqual(cdiv(-9, 5), -1)
-    self.assertEqual(cdiv(-4, 5), 0)
-    self.assertEqual(cdiv(0, 5), 0)
-    self.assertEqual(cdiv(4, 5), 0)
-    self.assertEqual(cdiv(9, 5), 1)
-  def test_div_neg(self):
-    self.assertEqual(cdiv(-9, -5), 1)
-    self.assertEqual(cdiv(-4, -5), 0)
-    self.assertEqual(cdiv(0, -5), 0)
-    self.assertEqual(cdiv(4, -5), 0)
-    self.assertEqual(cdiv(9, -5), -1)
-  def test_mod_pos(self):
-    self.assertEqual(cmod(-9, 5), -4)
-    self.assertEqual(cmod(-4, 5), -4)
-    self.assertEqual(cmod(0, 5), 0)
-    self.assertEqual(cmod(4, 5), 4)
-    self.assertEqual(cmod(9, 5), 4)
-  def test_mod_neg(self):
-    self.assertEqual(cmod(-9, -5), -4)
-    self.assertEqual(cmod(-4, -5), -4)
-    self.assertEqual(cmod(0, -5), 0)
-    self.assertEqual(cmod(4, -5), 4)
-    self.assertEqual(cmod(9, -5), 4)
-
-class TestGetBits(unittest.TestCase):
-  def test_low_bits(self):
-    self.assertEqual(getbits(0b11010110, 0, 3), 0b0110)
-
-  def test_high_bits(self):
-    self.assertEqual(getbits(0b11010110, 4, 7), 0b1101)
-
-  def test_middle_bits(self):
-    self.assertEqual(getbits(0b11010110, 3, 5), 0b010)
-
-  def test_full_range(self):
-    self.assertEqual(getbits(0b11010110, 0, 7), 0b11010110)
-
-  def test_single_bit(self):
-    self.assertEqual(getbits(0b100000000, 8, 8), 1)
-
-class TestArgFix(unittest.TestCase):
-  def test_none(self):
-    self.assertEqual(argfix(None), (None, ))
-    self.assertEqual(argfix(None, None), (None, None))
-  def test_positional_arguments(self):
-    self.assertEqual(argfix(1, 2, 3), (1, 2, 3))
-  def test_tuple(self):
-    self.assertEqual(argfix((1., 2., 3.)), (1., 2., 3.))
-  def test_list(self):
-    self.assertEqual(argfix([True, False]), (True, False))
-
-class TestWordWrap(unittest.TestCase):
-  def test_wrap_simple(self):
-    wrap = 10
-    st = "x"*wrap*2
-    st2 = word_wrap(st, wrap)
-    self.assertEqual(len(st2.splitlines()), 2)
-
-  def test_wrap_colored(self):
-    wrap = 10
-    st = colored("x"*wrap*2, "red")
-    st2 = word_wrap(st, wrap=wrap)
-    self.assertEqual(len(st2.splitlines()), 2)
-
-  def test_wrap_explicit_newline(self):
-    wrap = 10
-    st = "\n".join(["x"*wrap, "x"*wrap, "x"*wrap])
-    st2 = word_wrap(st, wrap=wrap)
-    self.assertEqual(len(st2.splitlines()), len(st.splitlines()))
-
-    st = "\n".join(["x"*(wrap+1), "x"*wrap, "x"*wrap])
-    st2 = word_wrap(st, wrap=wrap)
-    self.assertEqual(len(st2.splitlines()), len(st.splitlines())+1)
-
-    st = "\n".join(["x"*(wrap+1), "x"*(wrap+1), "x"*(wrap+1)])
-    st2 = word_wrap(st, wrap=wrap)
-    self.assertEqual(len(st2.splitlines()), len(st.splitlines())+3)
-
-class TestIsNumpyNdarray(unittest.TestCase):
-  def test_ndarray(self):
-    self.assertTrue(is_numpy_ndarray(np.array([1, 2, 3])))
-  def test_ndarray_tolist(self):
-    self.assertFalse(is_numpy_ndarray(np.array([1, 2, 3]).tolist()))
-  def test_list(self):
-    self.assertFalse(is_numpy_ndarray([1, 2, 3]))
-  def test_tensor(self):
-    self.assertFalse(is_numpy_ndarray(Tensor([1, 2, 3])))
-    self.assertFalse(is_numpy_ndarray(Tensor(np.array([1, 2, 3]))))
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_indexing.py

@@ -1,143 +0,0 @@
-# test cases are modified from pytorch test_indexing.py
-
-import unittest
-import numpy as np
-
-from tinygrad import Tensor, dtypes
-
-def numpy_testing_assert_equal_helper(a, b):
-  if isinstance(a, Tensor): a = a.numpy()
-  if isinstance(b, Tensor): b = b.numpy()
-  np.testing.assert_equal(a, b)
-
-class TestIndexing(unittest.TestCase):
-  def test_single_int(self):
-    v = Tensor.randn(5, 7, 3)
-    numpy_testing_assert_equal_helper(v[4].shape, (7, 3))
-
-  def test_multiple_int(self):
-    v = Tensor.randn(5, 7, 3)
-    numpy_testing_assert_equal_helper(v[4].shape, (7, 3))
-    numpy_testing_assert_equal_helper(v[4, :, 1].shape, (7,))
-
-  def test_none(self):
-    v = Tensor.randn(5, 7, 3)
-    numpy_testing_assert_equal_helper(v[None].shape, (1, 5, 7, 3))
-    numpy_testing_assert_equal_helper(v[:, None].shape, (5, 1, 7, 3))
-    numpy_testing_assert_equal_helper(v[:, None, None].shape, (5, 1, 1, 7, 3))
-    numpy_testing_assert_equal_helper(v[..., None].shape, (5, 7, 3, 1))
-
-  def test_int_indices(self):
-    v = Tensor.randn(5, 7, 3)
-    numpy_testing_assert_equal_helper(v[[0, 4, 2]].shape, (3, 7, 3))
-    numpy_testing_assert_equal_helper(v[:, [0, 4, 2]].shape, (5, 3, 3))
-    numpy_testing_assert_equal_helper(v[:, [[0, 1], [4, 3]]].shape, (5, 2, 2, 3))
-
-  def test_index_src_datatype(self):
-    src = Tensor.ones(3, 2, 4)
-    # test index
-    res = src[[0, 2, 1], :, :]
-    numpy_testing_assert_equal_helper(res.shape, src.shape)
-
-  def test_empty_slice(self):
-    x = Tensor.randn(2, 3, 4, 5)
-    y = x[:, :, :, 1]
-    z = y[:, 1:1, :]
-    numpy_testing_assert_equal_helper((2, 0, 4), z.shape)
-
-  def test_invalid_index(self):
-    x = Tensor.arange(0, 16).reshape(4, 4)
-    self.assertRaises(TypeError, lambda: x["0":"1"])
-
-  def test_out_of_bound_index(self):
-    x = Tensor.arange(0, 100).reshape(2, 5, 10)
-    self.assertRaises(IndexError, lambda: x[0, 5])
-    self.assertRaises(IndexError, lambda: x[4, 5])
-    self.assertRaises(IndexError, lambda: x[0, 1, 15])
-    self.assertRaises(IndexError, lambda: x[:, :, 12])
-
-  def test_take_along_dim(self):
-    # NOTE: the actual test logic is inside _test_against_numpy which is never called
-    # This test effectively does nothing but defines a function
-    def _test_against_numpy(t: Tensor, indices: Tensor, dim):
-      actual = t.gather(dim, indices)
-      t_np = t.numpy()
-      indices_np = indices.numpy()
-      expected = np.take_along_axis(t_np, indices_np, axis=dim)
-      numpy_testing_assert_equal_helper(actual, expected)
-
-      # TODO argsort
-      '''
-      for shape in [(3, 2), (2, 3, 5), (2, 4, 0), (2, 3, 1, 4)]:
-        for noncontiguous in [True, False]:
-          for dtype in (dtypes.float32, dtypes.int64):
-            t = make_tensor(shape, dtype=dtype, noncontiguous=noncontiguous)
-            for dim in list(range(t.ndim)) + [None]:
-              if dim is None:
-                indices = argsort(t.reshape(-1))
-              else:
-                indices = argsort(t, dim=dim)
-
-          _test_against_numpy(t, indices, dim)
-      '''
-
-      # test broadcasting
-      t = Tensor.ones((3, 4, 1))
-      indices = Tensor.ones((1, 2, 5), dtype=dtypes.int64)
-
-      _test_against_numpy(t, indices, 1)
-
-      # test empty indices
-      t = Tensor.ones((3, 4, 5))
-      indices = Tensor.ones((3, 0, 5), dtype=dtypes.int64)
-
-      _test_against_numpy(t, indices, 1)
-
-class TestNumpy(unittest.TestCase):
-  def test_index_no_floats(self):
-    a = Tensor([[[5.]]])
-
-    self.assertRaises(IndexError, lambda: a[0.0])
-    self.assertRaises(IndexError, lambda: a[0, 0.0])
-    self.assertRaises(IndexError, lambda: a[0.0, 0])
-    self.assertRaises(IndexError, lambda: a[0.0, :])
-    self.assertRaises(IndexError, lambda: a[:, 0.0])
-    self.assertRaises(IndexError, lambda: a[:, 0.0, :])
-    self.assertRaises(IndexError, lambda: a[0.0, :, :])
-    self.assertRaises(IndexError, lambda: a[0, 0, 0.0])
-    self.assertRaises(IndexError, lambda: a[0.0, 0, 0])
-    self.assertRaises(IndexError, lambda: a[0, 0.0, 0])
-    self.assertRaises(IndexError, lambda: a[-1.4])
-    self.assertRaises(IndexError, lambda: a[0, -1.4])
-    self.assertRaises(IndexError, lambda: a[-1.4, 0])
-    self.assertRaises(IndexError, lambda: a[-1.4, :])
-    self.assertRaises(IndexError, lambda: a[:, -1.4])
-    self.assertRaises(IndexError, lambda: a[:, -1.4, :])
-    self.assertRaises(IndexError, lambda: a[-1.4, :, :])
-    self.assertRaises(IndexError, lambda: a[0, 0, -1.4])
-    self.assertRaises(IndexError, lambda: a[-1.4, 0, 0])
-    self.assertRaises(IndexError, lambda: a[0, -1.4, 0])
-    # these two trigger slice internal type verification first
-    self.assertRaises(TypeError, lambda: a[0.0:, 0.0])
-    self.assertRaises(TypeError, lambda: a[0.0:, 0.0,:])
-
-  def test_none_index(self):
-    # `None` index adds newaxis
-    a = Tensor([1, 2, 3])
-    numpy_testing_assert_equal_helper(a[None].ndim, a.ndim+1)
-
-  def test_everything_returns_views(self):
-    # Before `...` would return a itself.
-    a = Tensor([5])
-
-    self.assertIs(a, a[()])
-    self.assertIs(a, a[...])
-    self.assertIs(a, a[:])
-
-  def test_broaderrors_indexing(self):
-    a = Tensor.zeros(5, 5)
-    self.assertRaises(IndexError, a.__getitem__, ([0, 1], [0, 1, 2]))
-    self.assertRaises(IndexError, a.contiguous().__setitem__, ([0, 1], [0, 1, 2]), 0)
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_llm_server.py

@@ -1,137 +0,0 @@
-import unittest, threading, time
-from unittest.mock import Mock
-
-class TestLLMServer(unittest.TestCase):
-  """Integration tests using the real OpenAI client."""
-
-  @classmethod
-  def setUpClass(cls):
-    cls.mock_tok = Mock()
-    cls.mock_tok.role = Mock(return_value=[100, 101])
-    cls.mock_tok.encode = Mock(return_value=[200, 201, 202])
-    cls.mock_tok.decode = Mock(return_value="Hello")
-    cls.mock_tok.end_turn = Mock(return_value=[998])
-
-    cls.mock_model = Mock()
-    cls.mock_model.generate = Mock(side_effect=lambda ids, **kwargs: iter([300, 301, 999]))
-
-    cls.bos_id = 1
-    cls.eos_id = 999
-
-    import tinygrad.apps.llm as llm_module
-    llm_module.model = cls.mock_model
-    llm_module.tok = cls.mock_tok
-    llm_module.bos_id = cls.bos_id
-    llm_module.eos_id = cls.eos_id
-
-    from tinygrad.apps.llm import Handler
-    from tinygrad.viz.serve import TCPServerWithReuse
-
-    cls.server = TCPServerWithReuse(('127.0.0.1', 0), Handler)
-    cls.port = cls.server.server_address[1]
-    cls.server_thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
-    cls.server_thread.start()
-    time.sleep(0.1)
-
-    from openai import OpenAI
-    cls.client = OpenAI(base_url=f"http://127.0.0.1:{cls.port}/v1", api_key="test")
-
-  @classmethod
-  def tearDownClass(cls):
-    cls.server.shutdown()
-    cls.server.server_close()
-
-  def test_chat_completion_stream(self):
-    stream = self.client.chat.completions.create(
-      model="test",
-      messages=[{"role": "user", "content": "Hello"}],
-      stream=True
-    )
-
-    chunks = list(stream)
-    self.assertGreater(len(chunks), 0)
-    self.assertEqual(chunks[0].choices[0].delta.role, "assistant")
-    self.assertEqual(chunks[-1].choices[0].finish_reason, "stop")
-
-  def test_openai_response_structure(self):
-    stream = self.client.chat.completions.create(
-      model="test-model",
-      messages=[{"role": "user", "content": "Test"}],
-      stream=True
-    )
-
-    for chunk in stream:
-      self.assertTrue(chunk.id.startswith("chatcmpl-"))
-      self.assertEqual(chunk.object, "chat.completion.chunk")
-      self.assertIsNotNone(chunk.choices)
-      self.assertIsNotNone(chunk.created)
-      self.assertIsInstance(chunk.created, int)
-      self.assertEqual(chunk.model, "test-model")
-
-  def test_stream_with_usage(self):
-    stream = self.client.chat.completions.create(
-      model="test",
-      messages=[{"role": "user", "content": "Hello"}],
-      stream=True,
-      stream_options={"include_usage": True}
-    )
-
-    chunks = list(stream)
-    last_chunk = chunks[-1]
-
-    self.assertIsNotNone(last_chunk.usage)
-    self.assertIsNotNone(last_chunk.usage.prompt_tokens)
-    self.assertIsNotNone(last_chunk.usage.completion_tokens)
-    self.assertIsNotNone(last_chunk.usage.total_tokens)
-
-  def test_multi_turn_conversation(self):
-    stream = self.client.chat.completions.create(
-      model="test",
-      messages=[
-        {"role": "system", "content": "You are helpful."},
-        {"role": "user", "content": "Hello"},
-        {"role": "assistant", "content": "Hi!"},
-        {"role": "user", "content": "How are you?"}
-      ],
-      stream=True
-    )
-
-    chunks = list(stream)
-    self.assertGreater(len(chunks), 0)
-    self.assertEqual(chunks[-1].choices[0].finish_reason, "stop")
-
-  def test_content_is_streamed(self):
-    stream = self.client.chat.completions.create(
-      model="test",
-      messages=[{"role": "user", "content": "Hello"}],
-      stream=True
-    )
-
-    contents = []
-    for chunk in stream:
-      if chunk.choices and chunk.choices[0].delta.content:
-        contents.append(chunk.choices[0].delta.content)
-
-    self.assertGreater(len(contents), 0)
-
-  def test_non_streaming(self):
-    resp = self.client.chat.completions.create(
-      model="test-model",
-      messages=[{"role": "user", "content": "Hello"}],
-      stream=False
-    )
-
-    self.assertTrue(resp.id.startswith("chatcmpl-"))
-    self.assertEqual(resp.object, "chat.completion")
-    self.assertEqual(resp.model, "test-model")
-    self.assertIsNotNone(resp.created)
-    self.assertEqual(len(resp.choices), 1)
-    self.assertEqual(resp.choices[0].message.role, "assistant")
-    self.assertIsNotNone(resp.choices[0].message.content)
-    self.assertEqual(resp.choices[0].finish_reason, "stop")
-    self.assertIsNotNone(resp.usage)
-    self.assertIsNotNone(resp.usage.prompt_tokens)
-    self.assertIsNotNone(resp.usage.completion_tokens)
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_multitensor.py

@@ -1,58 +0,0 @@
-import unittest
-from tinygrad import Tensor, GlobalCounters, dtypes
-
-class TestMultiRamUsage(unittest.TestCase):
-  def setUp(self):
-    self.baseline = GlobalCounters.mem_used
-    self.N = 100
-  def assertUsed(self, amt, strict=True):
-    used = GlobalCounters.mem_used - self.baseline
-    print(f"used {used} bytes")
-    if strict: self.assertEqual(used, amt)
-    else: self.assertLessEqual(used, amt)
-
-  def test_zeros(self):
-    _ = Tensor.zeros(self.N, self.N).contiguous().realize()
-    self.assertUsed(self.N*self.N*4)
-
-  def test_zeros_del(self):
-    _ = Tensor.zeros(self.N, self.N).contiguous().realize()
-    del _
-    self.assertUsed(0)
-
-  @unittest.skip("flaky")
-  def test_zeros_copy(self):
-    devices_2 = ("NULL:1", "NULL:2")
-    _ = Tensor.zeros(self.N, self.N).contiguous().to(devices_2).realize()
-    # NOTE: the first one on the DEFAULT device should be freed
-    self.assertUsed(self.N*self.N*4*2)
-
-  @unittest.skip("flaky")
-  def test_zeros_shard(self, devices=("NULL:1", "NULL:2")):
-    _ = Tensor.zeros(self.N, self.N).contiguous().shard(devices, axis=0).realize()
-    self.assertUsed(self.N*self.N*4) # sharding should not increase total ram usage
-  def test_zeros_shard_self(self): self.test_zeros_shard(("NULL:0", "NULL:1"))
-
-  @unittest.skip("flaky")
-  def test_zeros_contiguous_shard(self):
-    devices_2 = ("NULL:1", "NULL:2")
-    _ = Tensor.zeros(self.N, self.N).contiguous().shard(devices_2, axis=0).contiguous().realize()
-    self.assertUsed(self.N*self.N*4) # sharding should not increase total ram usage
-
-  def _test_matmul_half(self, dev_count:int):
-    N = 32
-    total_mem = {}
-    devs = tuple(f"NULL:{i}" for i in range(dev_count))
-    for dtype in {dtypes.float, dtypes.half}:
-      GlobalCounters.reset()
-      a = Tensor.empty((N, N), dtype=dtype, device=devs[0]).shard(devs, axis=0)
-      b = Tensor.empty((N, N), dtype=dtype, device=devs[0]).shard(devs, axis=None)
-      (a @ b).realize()
-      total_mem[dtype] = GlobalCounters.global_mem
-    self.assertEqual(total_mem[dtypes.half], total_mem[dtypes.float] // 2)
-
-  def test_matmul_half(self): self._test_matmul_half(dev_count=2)
-  def test_matmul_half_alt(self): self._test_matmul_half(dev_count=4)
-
-if __name__ == '__main__':
-  unittest.main()

test/null/test_rearrange_einops.py

@@ -1,171 +0,0 @@
-# modified from
-# https://github.com/arogozhnikov/einops/blob/master/tests/test_examples.py
-# https://github.com/arogozhnikov/einops/blob/master/tests/test_ops.py
-# https://github.com/arogozhnikov/einops/blob/master/tests/test_parsing.py
-
-import numpy as np
-import unittest
-from tinygrad import Tensor
-
-
-class test_rearrange_examples(unittest.TestCase):
-  def test1(self):
-    # transpose
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    y = x.rearrange("b c h w -> b h w c")
-    assert tuple(y.shape) == (10, 30, 40, 20)
-
-  def test2(self):
-    # view / reshape
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    y = x.rearrange("b c h w -> b (c h w)")
-    assert tuple(y.shape) == (10, 20 * 30 * 40)
-
-  def test3(self):
-    # depth-to-space
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    y = x.rearrange("b (c h1 w1) h w -> b c (h h1) (w w1)", h1=2, w1=2)
-    assert tuple(y.shape) == (10, 5, 30 * 2, 40 * 2)
-
-  def test4(self):
-    # space-to-depth
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    y = x.rearrange("b c (h h1) (w w1) -> b (h1 w1 c) h w", h1=2, w1=2)
-    assert tuple(y.shape) == (10, 20 * 4, 30 // 2, 40 // 2)
-
-  def test5(self):
-    # simple transposition
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    y = x.rearrange("b1 sound b2 letter -> b1 b2 sound letter")
-    assert tuple(y.shape) == (10, 30, 20, 40)
-
-  def test6(self):
-    # parsing parameters
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    t = x.rearrange("b c h w -> (b h w) c")
-    t = t[:, ::2]  # replacement for dot-product, just changes size of second axis
-    assert tuple(t.shape) == (10 * 30 * 40, 10)
-
-  def test7(self):
-    x = Tensor(np.arange(10 * 20 * 30 * 40, dtype=np.int32).reshape([10, 20, 30, 40]))
-    # split of embedding into groups
-    y1, y2 = x.rearrange("b (c g) h w -> g b c h w", g=2)
-    assert tuple(y1.shape) == (10, 10, 30, 40)
-    assert tuple(y2.shape) == (10, 10, 30, 40)
-
-  def test8(self):
-    x = Tensor(np.arange(10 * 20 * 1 * 1, dtype=np.int32).reshape([10, 20, 1, 1]))
-    # squeeze - unsqueeze
-    y = x.rearrange("b c () () -> b c")
-    assert tuple(y.shape) == (10, 20)
-    y = y.rearrange("b c -> c b () ()")
-    assert tuple(y.shape) == (20, 10, 1, 1)
-
-  def test9(self):
-    x = Tensor(np.arange(10 * 20 * 1 * 1, dtype=np.int32).reshape([10, 20, 1, 1]))
-    # squeeze - unsqueeze
-    y = x.rearrange("b c 1 1 -> b c")
-    assert tuple(y.shape) == (10, 20)
-    y = y.rearrange("b1 c -> c b1 1 1")
-    assert tuple(y.shape) == (20, 10, 1, 1)
-
-
-class test_rearrange_ops(unittest.TestCase):
-  def test_rearrange_errors(self):
-    x = Tensor.zeros([1, 1, 1, 1, 1])
-    x.rearrange("a b c d ... ->  a b c ... d")
-    bad_patterns = [
-      "a b c d (...) ->  a b c ... d",  # collapsed ellipsis on input
-      "a b (c d ... ->  a b c ... d",   # unbalanced brackets
-      "a b* c d ... ->  a b c ... d",   # not alphanumeric
-      "a b c d ->  a b c d -> a b c d", # two "->"
-      "a ... c ... ->  ... a ... c",    # two "..."
-      "a b c d e -> f b c d e",         # name mismatch
-    ]
-    for pattern in bad_patterns:
-      with self.assertRaises(AssertionError):
-        x.rearrange(pattern)
-
-    x.rearrange("... ->  (...)")
-    with self.assertRaises(AssertionError):
-      x.rearrange("(...) -> (...)")
-
-    y = Tensor.zeros([8, 1])
-    y.rearrange("(a1 a2 a3) b -> b a3 a2 a1", a1=2, a2=2)
-    with self.assertRaises(RuntimeError):
-      ## should fail as not enough dimensions specified
-      y.rearrange("(a1 a2 a3) b -> b a3 a2 a1", a1=2)
-    with self.assertRaises(ValueError):
-      ## should fail as 6 does not divide 8
-      y.rearrange("(a1 a2 a3) b -> b a3 a2 a1", a1=3, a2=2)
-    with self.assertRaises(AssertionError):
-      ## incorrect dimension provided for an axis that is only permuted
-      y.rearrange("(a1 a2 a3) b -> b a3 a2 a1", a1=2, a2=2, b=2)
-    with self.assertRaises(AssertionError):
-      ## unused axis provided
-      y.rearrange("(a b c) d -> a b c d", b=2, c=2, e=2)
-
-
-class test_rearrange_parsing(unittest.TestCase):
-  def test_elementary_axis_name(self):
-    for name in [
-      "a",
-      "b",
-      "h",
-      "dx",
-      "h1",
-      "zz",
-      "i9123",
-      "somelongname",
-      "Alex",
-      "camelCase",
-      "u_n_d_e_r_score",
-      "unreasonablyLongAxisName",
-    ]:
-      Tensor.ones((1,)).rearrange(f"{name} -> {name}")
-
-    for name in ["2b", "12", "_startWithUnderscore", "endWithUnderscore_", "_"]:
-      with self.assertRaises(AssertionError):
-        Tensor.ones((1,)).rearrange(f"{name} -> {name}")
-
-    with self.assertRaises(RuntimeError):
-      Tensor.ones((1,)).rearrange(" -> ")
-
-  def test_invalid_expressions(self):
-    # double ellipsis should raise an error
-    def _test_expression(expression: str):
-      Tensor.ones((2, 3, 4, 5, 6)).rearrange(f"{expression} -> {expression}")
-
-    _test_expression("... a b c d")
-    with self.assertRaises(AssertionError):
-      _test_expression("... a b c d ...")
-    with self.assertRaises(AssertionError):
-      _test_expression("... a b c (d ...)")
-    with self.assertRaises(AssertionError):
-      _test_expression("(... a) b c (d ...)")
-
-    # double/missing/enclosed parenthesis
-    Tensor.ones((2, 3, 4, 5, 6)).rearrange("a b c d ... -> (a) b c (d ...)")
-    with self.assertRaises(AssertionError):
-      _test_expression("(a)) b c (d ...)")
-    with self.assertRaises(AssertionError):
-      _test_expression("(a b c (d ...)")
-    with self.assertRaises(AssertionError):
-      _test_expression("(a) (()) b c (d ...)")
-    with self.assertRaises(AssertionError):
-      _test_expression("(a) ((b c) (d ...))")
-
-    # invalid identifiers
-    _test_expression("camelCase under_scored cApiTaLs ß ...")
-    with self.assertRaises(AssertionError):
-      _test_expression("1a")
-    with self.assertRaises(AssertionError):
-      _test_expression("_pre")
-    with self.assertRaises(AssertionError):
-      _test_expression("...pre")
-    with self.assertRaises(AssertionError):
-      _test_expression("pre...")
-
-
-if __name__ == "__main__":
-  unittest.main()