failing test from application

* assembly/amd: add ds perm instructions

* NO SKIP

* fix preexisting RDNA3 issues

* pcode

* assert

* asserts

* unify

* simp

* good fix

.github/actions/process-replay/action.yml

@@ -11,5 +11,5 @@ runs:
         git fetch origin $CURRENT_SHA
         export COMMIT_MESSAGE=$(git show -s --format=%B "$CURRENT_SHA")
         export CURRENT_HEAD=$(git rev-parse HEAD)
-        cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && IGNORE_OOB=1 PYTHONPATH=. python3 process_replay.py
+        cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git -c advice.detachedHead=false checkout origin/master && CHECK_OOB=0 PYTHONPATH=. python3 process_replay.py
         git checkout $CURRENT_HEAD  # restore to branch

.github/actions/setup-tinygrad/action.yml

@@ -56,7 +56,15 @@ runs:
 
     # **** Caching packages ****
 
+    - name: Cache Python packages (PR)
+      if: github.event_name == 'pull_request'
+      id: restore-venv-pr
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ github.workspace }}/.venv
+        key: venv-${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-${{ inputs.deps }}-${{ inputs.pydeps }}-${{ env.CACHE_VERSION }}
     - name: Cache Python packages
+      if: github.event_name != 'pull_request'
       id: restore-venv
       uses: actions/cache@v4
       with:
@@ -65,23 +73,23 @@ runs:
 
     # **** Caching downloads ****
 
-    - name: Cache downloads (Linux)
-      if: inputs.key != '' && runner.os == 'Linux'
-      uses: actions/cache@v4
+    - name: Cache downloads (PR)
+      if: inputs.key != '' && github.event_name == 'pull_request'
+      uses: actions/cache/restore@v4
       with:
-        path: ~/.cache/tinygrad/downloads/
+        path: ${{ runner.os == 'Linux' && '~/.cache/tinygrad/downloads/' || '~/Library/Caches/tinygrad/downloads/' }}
         key: downloads-${{ github.job }}-${{ inputs.key }}-${{ env.CACHE_VERSION }}
-    - name: Cache downloads (macOS)
-      if: inputs.key != '' && runner.os == 'macOS'
+    - name: Cache downloads
+      if: inputs.key != '' && github.event_name != 'pull_request'
       uses: actions/cache@v4
       with:
-        path: ~/Library/Caches/tinygrad/downloads/
+        path: ${{ runner.os == 'Linux' && '~/.cache/tinygrad/downloads/' || '~/Library/Caches/tinygrad/downloads/' }}
         key: downloads-${{ github.job }}-${{ inputs.key }}-${{ env.CACHE_VERSION }}
 
     # **** Python deps ****
 
     - name: Install dependencies in venv (with extra)
-      if: inputs.deps != '' && steps.restore-venv.outputs.cache-hit != 'true'
+      if: inputs.deps != '' && steps.restore-venv-pr.outputs.cache-hit != 'true' && steps.restore-venv.outputs.cache-hit != 'true'
       shell: bash
       run: |
         python -m venv .venv
@@ -92,7 +100,7 @@ runs:
         fi
         python -m pip install -e ".[${{ inputs.deps }}]" ${{ inputs.pydeps }} --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/
     - name: Install dependencies in venv (without extra)
-      if: inputs.deps == '' && steps.restore-venv.outputs.cache-hit != 'true'
+      if: inputs.deps == '' && steps.restore-venv-pr.outputs.cache-hit != 'true' && steps.restore-venv.outputs.cache-hit != 'true'
       shell: bash
       run: |
         python -m venv .venv
@@ -182,8 +190,14 @@ runs:
         echo "pkgs=$pkgs" >> "$GITHUB_OUTPUT"
         echo "hash=$(echo -n "$pkgs" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
 
+    - name: Cache apt (PR)
+      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true') && github.event_name == 'pull_request'
+      uses: actions/cache/restore@v4
+      with:
+        path: /var/cache/apt/archives/
+        key: ${{ runner.os }}-apt-${{ steps.apt-pkgs.outputs.hash }}-${{ env.CACHE_VERSION }}
     - name: Cache apt
-      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true')
+      if: runner.os == 'Linux' && (inputs.opencl == 'true' || inputs.amd == 'true' || inputs.cuda == 'true' || inputs.webgpu == 'true' || inputs.llvm == 'true') && github.event_name != 'pull_request'
       uses: actions/cache@v4
       with:
         path: /var/cache/apt/archives/
@@ -239,8 +253,17 @@ runs:
         ln -s /opt/homebrew/opt/[email protected] /opt/homebrew/opt/boost || true
         ln -s /opt/homebrew/opt/boost/lib/libboost_atomic-mt.dylib /opt/homebrew/opt/boost/lib/libboost_atomic.dylib || true
         ln -s /opt/homebrew/opt/boost/lib/libboost_thread-mt.dylib /opt/homebrew/opt/boost/lib/libboost_thread.dylib || true
+    - name: Cache gpuocelot (PR)
+      if: inputs.ocelot == 'true' && github.event_name == 'pull_request'
+      id: cache-build-pr
+      uses: actions/cache/restore@v4
+      env:
+        cache-name: cache-gpuocelot-build-1
+      with:
+        path: ${{ github.workspace }}/gpuocelot/ocelot
+        key: ${{ runner.os }}-gpuocelot-b16039dc940dc6bc4ea0a98380495769ff35ed99-rebuild-${{ env.CACHE_VERSION }}
     - name: Cache gpuocelot
-      if: inputs.ocelot == 'true'
+      if: inputs.ocelot == 'true' && github.event_name != 'pull_request'
       id: cache-build
       uses: actions/cache@v4
       env:
@@ -249,7 +272,7 @@ runs:
         path: ${{ github.workspace }}/gpuocelot/ocelot
         key: ${{ runner.os }}-gpuocelot-b16039dc940dc6bc4ea0a98380495769ff35ed99-rebuild-${{ env.CACHE_VERSION }}
     - name: Clone/compile gpuocelot
-      if: inputs.ocelot == 'true' && steps.cache-build.outputs.cache-hit != 'true'
+      if: inputs.ocelot == 'true' && steps.cache-build-pr.outputs.cache-hit != 'true' && steps.cache-build.outputs.cache-hit != 'true'
       shell: bash
       run: |
         git clone --recurse-submodules https://github.com/gpuocelot/gpuocelot.git ${{ github.workspace }}/gpuocelot

.github/workflows/autogen.yml

@@ -40,13 +40,13 @@ jobs:
         mesa: 'true'
         pydeps: 'pyyaml mako'
     - name: Install autogen support packages
-      run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev
+      run: sudo apt-get install -y --no-install-recommends libclang-20-dev llvm-20-dev hip-dev libusb-1.0-0-dev libdrm-dev
     - name: Regenerate autogen files
       run: |
-        find tinygrad/runtime/autogen -type f -name "*.py" -not -name "__init__.py" -not -name "comgr_3.py" -not -name "metal.py" -not -name "libclang.py" -delete
+        find tinygrad/runtime/autogen -type f -name "*.py" -not -name "__init__.py" -not -name "comgr_3.py" -not -name "metal.py" -not -name "iokit.py" -not -name "corefoundation.py" -not -name "libclang.py" -delete
         python3 -c "from tinygrad.runtime.autogen import opencl"
         python3 -c "from tinygrad.runtime.autogen import cuda, nvrtc, nvjitlink, nv_570, nv_580, nv"
-        python3 -c "from tinygrad.runtime.autogen import comgr, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd"
+        python3 -c "from tinygrad.runtime.autogen import comgr, hsa, hip, amd_gpu, sqtt, rocprof, amdgpu_kd, amdgpu_drm"
         python3 -c "from tinygrad.runtime.autogen.am import am, pm4_soc15, pm4_nv, sdma_4_0_0, sdma_5_0_0, sdma_6_0_0, smu_v13_0_0, smu_v13_0_6, smu_v14_0_2"
         python3 -c "from tinygrad.runtime.autogen import libc, kfd, io_uring, ib, pci, vfio"
         python3 -c "from tinygrad.runtime.autogen import llvm"
@@ -83,8 +83,8 @@ jobs:
         llvm: 'true'
     - name: Regenerate autogen files
       run: |
-        rm tinygrad/runtime/autogen/metal.py
-        LIBCLANG_PATH=/opt/homebrew/opt/llvm@20/lib/libclang.dylib python3 -c "from tinygrad.runtime.autogen import metal"
+        rm tinygrad/runtime/autogen/metal.py tinygrad/runtime/autogen/iokit.py tinygrad/runtime/autogen/corefoundation.py
+        python3 -c "from tinygrad.runtime.autogen import metal, iokit, corefoundation"
     - name: Check for differences
       run: |
         if ! git diff --quiet; then

.github/workflows/benchmark.yml

@@ -145,6 +145,10 @@ jobs:
       run: |
         echo "CACHEDB=/tmp/staging.db" >> $GITHUB_ENV
         rm -f /tmp/staging.db /tmp/staging.db-shm /tmp/staging.db-wal
+    - name: Kill stale pids
+      run: |
+        PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
+        PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids
     - name: UsbGPU boot time
       run: sudo -E PYTHONPATH=. DEBUG=2 AM_RESET=1 AMD=1 AMD_IFACE=USB time python3.11 test/test_tiny.py TestTiny.test_plus
     - name: UsbGPU tiny tests
@@ -332,9 +336,9 @@ jobs:
     - name: Setcap to python
       run: ./extra/amdpci/setup_python_cap.sh
     - name: Remove amd modules
-      run: ./extra/hcq/hcq_smi.py amd rmmod
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod
     - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py amd kill_pids
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
     #- name: Insert amdgpu
     #  run: sudo modprobe amdgpu
     - name: Symlink models and datasets
@@ -444,9 +448,9 @@ jobs:
     - name: Setcap to python
       run: ./extra/amdpci/setup_python_cap.sh
     - name: Remove amd modules
-      run: ./extra/hcq/hcq_smi.py amd rmmod
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod
     - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py amd kill_pids
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -496,9 +500,9 @@ jobs:
     - name: Setcap to python
       run: ./extra/amdpci/setup_python_cap.sh
     - name: Remove amd modules
-      run: ./extra/hcq/hcq_smi.py amd rmmod
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod
     - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py amd kill_pids
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -561,7 +565,7 @@ jobs:
     - name: openpilot compile3 0.10.1 driving_policy
       run: BENCHMARK_LOG=openpilot_0_10_1_policy PYTHONPATH="." ASSERT_MIN_STEP_TIME=3 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/driving_policy.onnx
     - name: openpilot compile3 0.10.1 dmonitoring
-      run: BENCHMARK_LOG=openpilot_0_10_1_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=10 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/dmonitoring_model.onnx
+      run: BENCHMARK_LOG=openpilot_0_10_1_dmonitoring PYTHONPATH="." ASSERT_MIN_STEP_TIME=11 DEV=QCOM FLOAT16=1 IMAGE=2 NOLOCALS=1 taskset -c 4-7 python3 examples/openpilot/compile3.py https://github.com/commaai/openpilot/raw/720392c9a5b986981fdbed1bb8c47a6c5573a50e/selfdrive/modeld/models/dmonitoring_model.onnx
     - name: benchmark MobileNetV2 on DSP
       run: |
         # generate quantized weights
@@ -587,9 +591,9 @@ jobs:
     - name: Setcap to python
       run: ./extra/amdpci/setup_python_cap.sh
     - name: Remove amd modules
-      run: ./extra/hcq/hcq_smi.py amd rmmod
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd rmmod
     - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py amd kill_pids
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py amd kill_pids
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -651,9 +655,9 @@ jobs:
     - name: Setcap to python
       run: ./extra/amdpci/setup_python_cap.sh
     - name: Remove nv modules
-      run: ./extra/hcq/hcq_smi.py nv rmmod
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py nv rmmod
     - name: Kill stale pids
-      run: ./extra/hcq/hcq_smi.py nv kill_pids
+      run: PYTHONPATH=. ./extra/hcq/hcq_smi.py nv kill_pids
     - name: Symlink models and datasets
       run: |
         mkdir -p weights

.github/workflows/test.yml

@@ -5,7 +5,7 @@ env:
   CAPTURE_PROCESS_REPLAY: 1
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PYTHONPATH: ${{ github.workspace }}
-  IGNORE_OOB: 0
+  CHECK_OOB: 1
 
 on:
   push:
@@ -26,7 +26,7 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: llvm-speed
-        deps: testing_minimal
+        deps: testing_unit
         llvm: 'true'
     - name: Speed Test
       run: CPU=1 CPU_LLVM=1 python3 test/speed/external_test_speed_v_torch.py
@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     env:
-      IGNORE_OOB: 1
+      CHECK_OOB: 0
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
@@ -98,7 +98,7 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: torch-backend-pillow-torchvision-et-pt
-        deps: testing_minimal
+        deps: testing_unit
         pydeps: "pillow torchvision expecttest"
         llvm: 'true'
     - name: Install ninja
@@ -134,7 +134,7 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: torch-backend-pillow-torchvision-et-pt
-        deps: testing_minimal
+        deps: testing_unit
         llvm: 'true'
     - name: Install ninja
       run: |
@@ -156,7 +156,7 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: be-minimal
-        deps: testing_minimal
+        deps: testing_unit
     - name: Test dtype with Python emulator
       run: DEBUG=1 PYTHON=1 python3 -m pytest -n=auto test/test_dtype.py test/test_dtype_alu.py
     - name: Test ops with Python emulator
@@ -229,19 +229,20 @@ jobs:
         deps: linting
     - name: Lint bad-indentation and trailing-whitespace with pylint
       run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string='  ' --recursive=y .
-    - name: Lint with ruff
+    - name: Run pre-commit linting hooks
+      run: SKIP=tiny,tests,example pre-commit run --all-files
+    - name: Lint additional files with ruff
       run: |
-        pip3 install --upgrade --force-reinstall ruff==0.14.10
-        pre-commit run ruff --all-files
         python3 -m ruff check examples/mlperf/ --ignore E501
         python3 -m ruff check extra/thunder/tiny/ --ignore E501 --ignore F841 --ignore E722
         python3 -m ruff check extra/torch_backend/backend.py
-    - name: Run mypy
+    - name: Run mypy with lineprecision report
       run: |
         python -m mypy --lineprecision-report .
+        grep -v autogen lineprecision.txt | awk 'NR>2 {lines+=$2; precise+=$3; imprecise+=$4; any+=$5; empty+=$6} END {t=lines-empty; printf "TOTAL: %d lines, %d precise (%.1f%%), %d imprecise (%.1f%%), %d any (%.1f%%)\n", t, precise, 100*precise/t, imprecise, 100*imprecise/t, any, 100*any/t}'
         cat lineprecision.txt
     - name: Run TYPED=1
-      run: TYPED=1 python -c "import tinygrad"
+      run: CHECK_OOB=0 DEV=CPU TYPED=1 python test/test_tiny.py
 
   unittest:
     name: Unit Tests
@@ -254,10 +255,13 @@ jobs:
     - name: Setup Environment
       uses: ./.github/actions/setup-tinygrad
       with:
-        key: unittest-12
-        pydeps: "pillow numpy ftfy regex"
+        key: unittest-13
+        pydeps: "pillow ftfy regex pre-commit"
         deps: testing_unit
         llvm: 'true'
+        amd: 'true'
+    - name: Run pre-commit test hooks
+      run: SKIP=ruff,mypy pre-commit run --all-files
     - name: Check Device.DEFAULT
       run: python -c "from tinygrad import Device; assert Device.DEFAULT == 'CPU', Device.DEFAULT"
     - name: Run unit tests
@@ -344,7 +348,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: gpu-image
-          deps: testing_minimal
+          deps: testing_unit
           opencl: 'true'
       - name: Test CL IMAGE=2 ops
         run: |
@@ -365,7 +369,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: gen-dataset
-          deps: testing_minimal
+          deps: testing
           opencl: 'true'
       - name: Generate Dataset
         run: CL=1 extra/optimization/generate_dataset.sh
@@ -420,7 +424,7 @@ jobs:
         with:
           key: onnxoptc
           deps: testing
-          python-version: '3.11'
+          python-version: '3.12'
           llvm: 'true'
       - name: Test ONNX (CPU)
         run: CPU=1 CPU_LLVM=0 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
@@ -448,7 +452,7 @@ jobs:
           key: onnxoptl
           deps: testing
           pydeps: "tensorflow==2.19"
-          python-version: '3.11'
+          python-version: '3.12'
           opencl: 'true'
       - name: Test ONNX (CL)
         run: CL=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
@@ -474,7 +478,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     env:
-      IGNORE_OOB: 1
+      CHECK_OOB: 0
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -522,7 +526,7 @@ jobs:
       with:
         key: metal
         deps: testing
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Test models (Metal)
       run: METAL=1 python -m pytest -n=auto test/models --durations=20
     - name: Test LLaMA compile speed
@@ -541,7 +545,7 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: devectorize-minimal
-        deps: testing_minimal
+        deps: testing_unit
         pydeps: "pillow"
         llvm: "true"
     - name: Test LLVM=1 DEVECTORIZE=0
@@ -562,8 +566,8 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: dsp-minimal
-        deps: testing_minimal
-        pydeps: "onnx==1.18.0 onnxruntime pillow"
+        deps: testing_unit
+        pydeps: "onnx==1.18.0 onnxruntime"
         llvm: "true"
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
@@ -596,8 +600,8 @@ jobs:
       uses: ./.github/actions/setup-tinygrad
       with:
         key: webgpu-minimal
-        deps: testing_minimal
-        python-version: '3.11'
+        deps: testing_unit
+        python-version: '3.12'
         webgpu: 'true'
     - name: Check Device.DEFAULT (WEBGPU) and print some source
       run: |
@@ -630,7 +634,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: ${{ matrix.backend }}-minimal
-          deps: testing_minimal
+          deps: testing_unit
           amd: 'true'
           llvm: ${{ matrix.backend == 'amdllvm' && 'true' }}
       - name: Check Device.DEFAULT and print some source
@@ -648,16 +652,23 @@ jobs:
         run: TRANSCENDENTAL=2 python -m pytest -n=auto test/test_ops.py::TestOps::test_sin test/test_ops.py::TestOps::test_cos test/test_ops.py::TestOps::test_tan test/test_ops.py::TestOps::test_exp test/test_ops.py::TestOps::test_log --durations=20
       - name: Run TestOps.test_add with SQTT
         run: |
-          VIZ=1 PMC=1 DEBUG=5 python3 test/test_ops.py TestOps.test_add
-          VIZ=1 SQTT=1 DEBUG=5 python3 test/test_ops.py TestOps.test_add
+          VIZ=-2 DEBUG=5 python3 test/test_ops.py TestOps.test_add
           extra/sqtt/rgptool.py create "/tmp/profile.pkl.$USER" -o /tmp/gpu0.rgp
+      - name: Run AMD emulated mmapeak on NULL backend
+        env:
+          AMD: 0
+        run: PYTHONPATH=. NULL=1 EMULATE=AMD python extra/mmapeak/mmapeak.py
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
 
   testamdasm:
     name: AMD ASM IDE
     runs-on: ubuntu-24.04
-    timeout-minutes: 10
+    timeout-minutes: 20
+    env:
+      AMD: 1
+      PYTHON_REMU: 1
+      MOCKGPU: 1
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -665,12 +676,12 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: rdna3-emu
-          deps: testing_minimal
+          deps: testing_unit
           amd: 'true'
-          python-version: '3.13'
+          python-version: '3.14'
       - name: Verify AMD autogen is up to date
         run: |
-          python -m extra.assembly.amd.amdxml
+          python -m extra.assembly.amd.generate
           git diff --exit-code extra/assembly/amd/autogen/
       - name: Install LLVM 21
         run: |
@@ -683,16 +694,16 @@ jobs:
       - name: Install rocprof-trace-decoder
         run: sudo PYTHONPATH="." ./extra/sqtt/install_sqtt_decoder.py
       - name: Run RDNA3 emulator tests
-        run: python -m pytest -n=auto extra/assembly/amd/ --durations 20
+        run: AMD_LLVM=0 python -m pytest -n=auto extra/assembly/amd/ --durations 20
       - name: Run RDNA3 emulator tests (AMD_LLVM=1)
         run: AMD_LLVM=1 python -m pytest -n=auto extra/assembly/amd/ --durations 20
       - name: Run RDNA3 dtype tests
-        run: AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=0 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py
+        run: AMD_LLVM=0 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py --durations 20
       - name: Run RDNA3 dtype tests (AMD_LLVM=1)
-        run: AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=1 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py
+        run: AMD_LLVM=1 pytest -n=auto test/test_dtype_alu.py test/test_dtype.py --durations 20
       # TODO: run all once emulator is faster
       - name: Run RDNA3 ops tests
-        run: SKIP_SLOW_TEST=1 AMD=1 PYTHON_REMU=1 MOCKGPU=1 AMD_LLVM=0 pytest -n=auto test/test_ops.py -k "test_sparse_categorical_crossentropy or test_tril"
+        run: SKIP_SLOW_TEST=1 AMD_LLVM=0 pytest -n=auto test/test_ops.py -k "test_sparse_categorical_crossentropy or test_tril or test_nonzero or test_softmax_argmax" --durations 20
 
   testnvidia:
     strategy:
@@ -713,7 +724,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: ${{ matrix.backend }}-minimal
-          deps: testing_minimal
+          deps: testing_unit
           cuda: 'true'
           ocelot: 'true'
       - name: Set env
@@ -725,6 +736,8 @@ jobs:
       - name: Run pytest (cuda)
         # skip multitensor because it's slow
         run: python -m pytest -n=auto test/ --ignore=test/models --ignore=test/unit --ignore test/test_gc.py --ignore test/test_multitensor.py --durations=20
+      - name: Run TestOps.test_add with PMA
+        run: VIZ=-1 PMA=1 DEBUG=5 python3 test/test_ops.py TestOps.test_add
       - name: Run process replay tests
         uses: ./.github/actions/process-replay
 
@@ -744,7 +757,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: ${{ matrix.backend }}-minimal
-          deps: testing_minimal
+          deps: testing_unit
           opencl: ${{ matrix.backend == 'opencl' && 'true' }}
           llvm: ${{ matrix.backend == 'llvm' || matrix.backend == 'lvp' }}
           mesa: ${{ matrix.backend == 'lvp' && 'true' }}
@@ -775,14 +788,12 @@ jobs:
       with:
         key: metal
         deps: testing
-        python-version: '3.11'
+        python-version: '3.12'
         amd: 'true'
         cuda: 'true'
         ocelot: 'true'
         llvm: 'true'
     - name: Run unit tests
-      env:
-        LIBCLANG_PATH: '/opt/homebrew/opt/llvm@20/lib/libclang.dylib'
       run: METAL=1 python -m pytest -n=auto test/unit/ --durations=20
     - name: Run ONNX
       run: METAL=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
@@ -875,8 +886,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: macos-${{ matrix.backend }}-minimal
-          deps: testing_minimal
-          pydeps: "capstone"
+          deps: testing_unit
           llvm: ${{ matrix.backend == 'llvm' || matrix.backend == 'lvp' }}
           mesa: ${{ matrix.backend == 'lvp' && 'true' }}
       - name: Set env
@@ -943,7 +953,7 @@ jobs:
         uses: ./.github/actions/setup-tinygrad
         with:
           key: compile-${{ matrix.backend }}
-          deps: testing_minimal
+          deps: testing_unit
           mesa: ${{ (matrix.backend == 'ir3' || matrix.backend == 'nak') && 'true' }}
           python-version: '3.14'
       - name: Set env

.gitignore

@@ -58,10 +58,11 @@ weights
 *.lprof
 comgr_*
 *.pkl
+!extra/sqtt/examples/**/*.pkl
 site/
 profile_stats
 *.log
 target
 .mypy_cache
 mutants
-.mutmut-cache
+.mutmut-cache

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
         pass_filenames: false
       - id: tests
         name: comprehensive test suite
-        entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/test_ops.py test/test_schedule.py test/test_assign.py test/test_tensor.py test/test_jit.py test/unit/test_schedule_cache.py test/unit/test_pattern_matcher.py test/unit/test_uop_symbolic.py test/unit/test_helpers.py
+        entry: env OMP_NUM_THREADS=1 SKIP_SLOW_TEST=1 PYTHONPATH="." python3 -m pytest -n=6 test/test_ops.py test/test_schedule.py test/unit/test_assign.py test/test_tensor.py test/test_jit.py test/unit/test_schedule_cache.py test/unit/test_pattern_matcher.py test/unit/test_uop_symbolic.py test/unit/test_helpers.py
         language: system
         always_run: true
         pass_filenames: false

examples/anthropic_challenge.py

@@ -0,0 +1,196 @@
+from tinygrad import Tensor, dtypes, Context, getenv, UOp, fetch
+from tinygrad.uop.ops import Ops, PatternMatcher, UPat
+from tinygrad.uop.symbolic import symbolic
+from tinygrad.codegen import Renderer
+from tinygrad.codegen.opt import Opt, OptOps
+
+# ************************* implementation of the problem ************************
+
+def myhash(a: Tensor) -> Tensor:
+  a = (a + 0x7ED55D16) + (a << 12)
+  a = (a ^ 0xC761C23C) ^ (a >> 19)
+  a = (a + 0x165667B1) + (a << 5)
+  a = (a + 0xD3A2646C) ^ (a << 9)
+  a = (a + 0xFD7046C5) + (a << 3)
+  a = (a ^ 0xB55A4F09) ^ (a >> 16)
+  return a
+
+def select_with_where_tree(values: Tensor, relative_idx: Tensor) -> Tensor:
+  n = values.shape[0]
+  if n == 1: return values[0].expand(relative_idx.shape)
+
+  mid = n // 2
+  left = select_with_where_tree(values[:mid], relative_idx)
+  right = select_with_where_tree(values[mid:], relative_idx - mid)
+
+  go_left = relative_idx < mid
+  return go_left.where(left, right)
+
+def tree_traversal(forest: Tensor, val: Tensor, height: int, rounds: int, where_tree_threshold=3) -> Tensor:
+  # All walkers start at idx=0
+  idx = Tensor.zeros(val.shape, device=val.device, dtype=dtypes.uint32)
+
+  for r in range(rounds):
+    level = r % (height + 1)
+    level_start = (1 << level) - 1
+    level_size = 1 << level
+
+    if level == 0:
+      # At root (level 0), all walkers are at idx=0
+      # No gather needed, just broadcast the root value
+      node_val = forest[0].expand(val.shape)
+      idx = idx * 0  # Reset to 0
+    elif level <= where_tree_threshold:
+      # Small level: use where-tree
+      level_values = forest[level_start : level_start + level_size]
+      relative_idx = (idx - level_start)
+      node_val = select_with_where_tree(level_values, relative_idx)
+    else:
+      # Large level: use gather
+      node_val = forest.gather(0, idx)
+
+    val = myhash(val ^ node_val)
+    idx = (idx << 1) + (1 + (val & 1))
+
+    # No wrap check needed! At round 10 (level becomes 0), we reset idx above.
+
+  return val.contiguous(arg=(Opt(OptOps.UPCAST, 0, 8),))
+
+# ************************* renderer for VLIW machine *************************
+
+def loop_unrolling(sink:UOp):
+  rng = [x for x in sink.toposort() if x.op is Ops.RANGE]
+  if len(rng) == 0: return None
+  print(f"unrolling loop with size {rng[0].vmax+1}")
+  unrolled_sinks = [sink.substitute({rng[0]:rng[0].const_like(i)}).src[0] for i in range(rng[0].vmax+1)]
+  return UOp.sink(*unrolled_sinks, arg=sink.arg)
+
+global_addrs = []
+vliw_prepare = PatternMatcher([
+  # loop unrolling (should be a part of tinygrad)
+  (UPat(Ops.SINK, name="sink"), loop_unrolling),
+  # cast is fake
+  (UPat(Ops.CAST, name="c"), lambda c: c.src[0]),
+  # rewrites to hardcode the addresses in memory
+  (UPat(Ops.DEFINE_GLOBAL, name="dg"), lambda dg: UOp.const(dtypes.uint, global_addrs[dg.arg])),
+  # INDEX is just plus
+  (UPat(Ops.INDEX, name="i"), lambda i: i.src[0]+i.src[1]),
+])+symbolic
+
+class VLIWRenderer(Renderer):
+  has_local = False  # TODO: this should be the default / cleaned up
+  # this says this backend supports MULACC + more. decompositions uses this
+  code_for_op: dict = {Ops.MULACC: None, Ops.ADD: "+", Ops.MUL: "*",
+                       Ops.XOR: "^", Ops.AND: "&", Ops.OR: "|",
+                       Ops.SHL: "<<", Ops.SHR: ">>", Ops.CMPLT: "<"}
+  # this matcher runs while still in graph form
+  pre_matcher = vliw_prepare
+
+  def render(self, uops:list[UOp]):
+
+    # TODO: this is a minimal renderer. for low cycle count, make it good
+    # to get speed, you need to add VLIW packing
+    # to get under 1536 regs, you need to add a register allocator
+    # we left the fun parts to you
+
+    print(f"rendering with {len(uops)} uops")
+    reg, inst = 0, []
+    r: dict[UOp, int] = {}
+    for u in uops:
+      assert u.dtype.count in (1,8), "dtype count must be 1 or 8"
+
+      # dumb register allocator
+      if u.op not in {Ops.STORE, Ops.SINK, Ops.GEP}:
+        r[u] = reg
+        reg += u.dtype.count
+
+      # render UOps to instructions
+      match u.op:
+        case Ops.SINK:
+          inst.append({"flow": [("halt",)]})
+        case Ops.CONST:
+          inst.append({"load": [("const", r[u], u.arg)]})
+        case Ops.GEP:
+          # a GEP is just an alias to a special register in the vector
+          r[u] = r[u.src[0]] + u.arg[0]
+        case Ops.VECTORIZE:
+          if all(s == u.src[0] for s in u.src):
+            # if all sources are the same, we can broadcast
+            inst.append({"valu": [("vbroadcast", r[u], r[u.src[0]])]})
+          else:
+            # this is a copy into a contiguous chunk of registers
+            inst.extend({"flow": [("add_imm", r[u]+i, r[s], 0)]} for i,s in enumerate(u.src) if r[s] != r[u]+i)
+        case Ops.LOAD:
+          op = "vload" if u.dtype.count > 1 else "load"
+          inst.append({"load": [(op, r[u], r[u.src[0]])]})
+        case Ops.STORE:
+          op = "vstore" if u.src[1].dtype.count > 1 else "store"
+          inst.append({"store": [(op, r[u.src[0]], r[u.src[1]])]})
+        case Ops.MULACC:
+          assert u.dtype.count == 8
+          inst.append({"valu": [("multiply_add", r[u], r[u.src[0]], r[u.src[1]], r[u.src[2]])]})
+        case Ops.WHERE:
+          assert u.dtype.count == 8
+          inst.append({"flow": [("vselect", r[u], r[u.src[0]], r[u.src[1]], r[u.src[2]])]})
+        case _ if u.op in self.code_for_op:
+          cat = "valu" if u.dtype.count > 1 else "alu"
+          inst.append({cat: [(self.code_for_op[u.op], r[u], r[u.src[0]], r[u.src[1]])]})
+        case _:
+          raise NotImplementedError(f"unhandled op {u.op}")
+    return repr(inst)
+
+# ************************* test and render *************************
+
+import sys, types
+PROBLEM_URL = "https://raw.githubusercontent.com/anthropics/original_performance_takehome/refs/heads/main/tests/frozen_problem.py"
+sys.modules["problem"] = problem = types.ModuleType("problem")
+exec(fetch(PROBLEM_URL).read_text(), problem.__dict__)
+
+if __name__ == "__main__":
+  batch_size = getenv("BS", 256)
+  height = 10
+  rounds = getenv("ROUNDS", 16)
+
+  # build problem
+  tree = problem.Tree.generate(height)
+  inp = problem.Input.generate(tree, batch_size, rounds)
+  mem = problem.build_mem_image(tree, inp)
+  global_addrs.extend([mem[6], mem[6], mem[4]])  # output, input, forest
+
+  # *** verify the kernel in tinygrad compared to reference ***
+
+  forest_t = Tensor(tree.values, dtype=dtypes.uint32)
+  val_t = Tensor(inp.values, dtype=dtypes.uint32)
+
+  if getenv("VERIFY", 1):
+    # verify on normal tinygrad device
+    with Context(PCONTIG=2):
+      out = tree_traversal(forest_t, val_t, height, rounds)
+      val_out = out.tolist()
+    problem.reference_kernel(tree, inp)
+    assert val_out == inp.values
+    print("verification passed")
+
+  # *** render to device ***
+
+  from tinygrad.codegen import get_program
+  with Context(PCONTIG=2, DEVECTORIZE=2, SPEC=0):
+    out = tree_traversal(forest_t, val_t, height, rounds)
+    sink = out.schedule()[-1].ast
+    prg = get_program(sink, VLIWRenderer())
+
+  # *** run on Machine and compare ***
+
+  # NOTE: the scratch size needs to be reduced to 1536 when you have a register allocator
+  src = eval(prg.src)
+  max_regs = max(t[1] for instr in src for v in instr.values() for t in v if len(t) > 1) + 8
+  print(f"{max_regs:5d} regs used" + ("" if max_regs <= 1536 else "       <-- WARNING: TOO MANY REGISTERS, MUST BE <= 1536"))
+  machine = problem.Machine(mem, src, problem.DebugInfo(scratch_map={}), n_cores=1, trace=False, scratch_size=max_regs)
+  machine.run()
+  print(f"ran for {machine.cycle:5d} cycles" + ("" if machine.cycle <= 1363 else "  <-- EVEN CLAUDE GOT 1363"))
+
+  # compare to reference
+  ref_mem = mem.copy()
+  for _ in problem.reference_kernel2(ref_mem, {}): pass
+  assert machine.mem[mem[6]:mem[6]+mem[2]] == ref_mem[mem[6]:mem[6]+mem[2]]
+  print("compare passed!")

examples/audio_helpers.py

@@ -0,0 +1,79 @@
+from typing import Optional
+from tinygrad import Tensor
+from tinygrad.dtype import DTypeLike, dtypes
+import math
+
+# rewritten from numpy
+def rfftfreq(n: int, d: float = 1.0, device=None) -> Tensor:
+  val = 1.0 / (n * d)
+  N = n // 2 + 1
+  results = Tensor.arange(N, device=device)
+  return results * val
+
+# just like in librosa
+def fft_frequencies(sr: float, n_fft: int) -> Tensor:
+  return rfftfreq(n=n_fft, d=1.0 / sr)
+
+def hz_to_mel(freq: Tensor) -> Tensor:
+  # linear part
+  f_min = 0.0
+  f_sp = 200.0 / 3
+  mels = (freq - f_min) / f_sp
+
+  # log-scale part
+  min_log_hz = 1000.0  # beginning of log region (Hz)
+  mask = freq >= min_log_hz
+  return mask.where(((min_log_hz - f_min) / f_sp) + (freq / min_log_hz).log() / (math.log(6.4) / 27.0), mels)
+
+def mel_to_hz(mels: Tensor) -> Tensor:
+  # linear scale
+  f_min = 0.0
+  f_sp = 200.0 / 3
+  freqs = f_min + f_sp * mels
+
+  # nonlinear scale
+  min_log_hz = 1000.0  # beginning of log region (Hz)
+  min_log_mel = (min_log_hz - f_min) / f_sp  # same (Mels)
+  logstep = math.log(6.4) / 27.0  # step size for log region
+
+  log_t = mels >= min_log_mel
+  freqs = log_t.where(min_log_hz * ((logstep * (mels - min_log_mel)).exp()), freqs)
+  return freqs
+
+def mel_frequencies(n_mels: int = 128, *, fmin: float = 0.0, fmax: float = 11025.0) -> Tensor:
+  # center freqs of mel bands - uniformly spaced between limits
+  min_max_mel = hz_to_mel(Tensor([fmin, fmax]))
+
+  mels = Tensor.linspace(min_max_mel[0], min_max_mel[1], n_mels)
+  hz = mel_to_hz(mels)
+  return hz
+
+def mel(
+  *,
+  sr: float,
+  n_fft: int,
+  n_mels: int = 128,
+  fmin: float = 0.0,
+  fmax: Optional[float] = None,
+  dtype: DTypeLike = dtypes.default_float,
+) -> Tensor:
+  if fmax is None:
+    fmax = float(sr) / 2
+
+  n_mels = int(n_mels)
+
+  fftfreqs = fft_frequencies(sr=sr, n_fft=n_fft)  # center freqs of each FFT bin
+  mel_f = mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax)  # center freqs of mel bands
+
+  fdiff = mel_f[1:] - mel_f[:-1]
+  ramps = mel_f[None].T.expand(-1, fftfreqs.shape[-1]) - fftfreqs
+
+  lower = -ramps[:n_mels] / fdiff[:n_mels][None].T
+  upper = ramps[2 : n_mels + 2] / fdiff[1 : n_mels + 1][None].T
+  weights = lower.minimum(upper).maximum(0)
+
+  # Slaney-style mel is scaled to be approx constant energy per channel
+  enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels])
+  weights *= enorm[:, None]
+
+  return weights

examples/mlperf/model_train.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import multiprocessing
 
 from tinygrad import Device, GlobalCounters, Tensor, TinyJit, dtypes
-from tinygrad.helpers import getenv, BEAM, WINO, round_up, diskcache_clear, Profiling
+from tinygrad.helpers import getenv, BEAM, WINO, round_up, diskcache_clear, Profiling, profile_marker
 from tinygrad.nn.state import get_parameters, get_state_dict, load_state_dict, safe_load, safe_save
 from tinygrad.nn.optim import LAMB, LARS, SGD, OptimizerGroup, Adam, AdamW
 
@@ -1286,17 +1286,23 @@ def train_llama3():
   from examples.llama3 import MODEL_PARAMS
   from examples.mlperf.lr_schedulers import CosineAnnealingLRWithWarmup
 
+  BENCHMARK = getenv("BENCHMARK")
+
   config = {}
   BASEDIR            = config["BASEDIR"]                = Path(getenv("BASEDIR", "/raid/datasets/c4/"))
   BS                 = config["BS"]                     = getenv("BS", 16)
   grad_acc           = config["GRADIENT_ACC_STEPS"]     = getenv("GRADIENT_ACC_STEPS", 1)
-  assert grad_acc == 1, f"{grad_acc=} is not supported"
   GBS                = config["GLOBAL_BATCH_SIZE"]      = BS * grad_acc
   SEED               = config["SEED"]                   = getenv("SEED", 5760)
   SEQLEN             = config["SEQLEN"]                 = getenv("SEQLEN", 8192)
   TRAIN_ON_VAL       = config["TRAIN_ON_VAL"]           = getenv("TRAIN_ON_VAL", 0)
   SMALL              = config["SMALL"]                  = getenv("SMALL", 0)
   SAMPLES            = config["SAMPLES"]                = getenv("SAMPLES", 5_760 if TRAIN_ON_VAL else 1_200_000 * 1152)
+  EVAL_SAMPLES       = config["EVAL_SAMPLES"]           = getenv("EVAL_SAMPLES", 5760 if not SMALL else 1024)
+  MAX_STEPS          = config["MAX_STEPS"]              = getenv("MAX_STEPS", math.ceil(1_200_000 * 1152 / GBS))
+  WARMUP_STEPS       = config["WARMUP_STEPS"]           = getenv("WARMUP_STEPS", math.ceil(8000 * 1152 / GBS))
+  LR                 = config["LR"]                     = getenv("LR", 8e-5 * GBS / 1152)
+  END_LR             = config["END_LR"]                 = getenv("END_LR", 8e-7)
   EVAL_FREQ          = config["EVAL_FREQ"]              = getenv("EVAL_FREQ", 46080)
   EVAL_BS            = config["EVAL_BS"]                = getenv("EVAL_BS", 16)
   EVAL_TARGET        = config["EVAL_TARGET"]            = getenv("EVAL_TARGET", 5.6)
@@ -1310,10 +1316,12 @@ def train_llama3():
   opt_adamw_weight_decay = 0.1
 
   opt_gradient_clip_norm = 1.0
-  opt_learning_rate_warmup_steps = getenv("WARMUP_STEPS", math.ceil(8000 * 1152 / GBS))
-  opt_learning_rate_decay_steps = getenv("MAX_STEPS", math.ceil(1_200_000 * 1152 / GBS)) - opt_learning_rate_warmup_steps
-  opt_base_learning_rate = getenv("LR", 8e-5 * GBS / 1152)  # NOTE: cannot change for benchmark
-  opt_end_learning_rate = getenv("END_LR", 8e-7)
+  opt_learning_rate_warmup_steps = WARMUP_STEPS
+  opt_learning_rate_decay_steps = MAX_STEPS - opt_learning_rate_warmup_steps
+  opt_base_learning_rate = LR
+  opt_end_learning_rate = END_LR
+
+  Tensor.manual_seed(SEED)  # seed for weight initialization
 
   # ** init wandb **
   WANDB = getenv("WANDB")
@@ -1326,6 +1334,8 @@ def train_llama3():
   # vocab_size from the mixtral tokenizer
   if not SMALL: model_params |= {"vocab_size": 32000}
   if (llama_layers:=getenv("LLAMA_LAYERS")) != 0: model_params['n_layers'] = llama_layers
+  print(f"model parameters: {model_params}")
+
   model = Transformer(**model_params, max_context=SEQLEN, jit=False, disable_kv_cache=True)
   params = get_parameters(model)
   # weights are all bfloat16 for now
@@ -1361,6 +1371,12 @@ def train_llama3():
 
   optim = AdamW(get_parameters(model), lr=0.0,
                 b1=opt_adamw_beta_1, b2=opt_adamw_beta_2, eps=opt_adamw_epsilon, weight_decay=opt_adamw_weight_decay)
+
+  # init grads
+  for p in optim.params:
+    p.grad = p.zeros_like().contiguous().realize()
+  grads = [p.grad for p in optim.params]
+
   scheduler = CosineAnnealingLRWithWarmup(optim, opt_base_learning_rate, opt_end_learning_rate, opt_learning_rate_warmup_steps, opt_learning_rate_decay_steps)
 
   if resume_ckpt := getenv("RESUME_CKPT"):
@@ -1373,9 +1389,7 @@ def train_llama3():
     load_state_dict(scheduler, safe_load(fn), realize=False)
 
   @TinyJit
-  @Tensor.train()
-  def train_step(model, tokens:Tensor):
-    optim.zero_grad()
+  def minibatch(tokens:Tensor):
     if (DP := getenv("DP", 1)) > 1:
       device = tuple(f"{Device.DEFAULT}:{i}" for i in range(DP))
       tokens = tokens.shard(device, 0)
@@ -1385,27 +1399,40 @@ def train_llama3():
     logits:Tensor = model(tokens[:, :-1], start_pos=0, temperature=math.nan)
     loss = logits.sparse_categorical_crossentropy(tokens[:, 1:])
     loss.backward()
+    assert all(p.grad is g for p,g in zip(optim.params, grads))
+    Tensor.realize(loss, *grads)
+    return loss
+
+  @TinyJit
+  def optim_step():
+    for p in optim.params:
+      p.grad.assign(p.grad / grad_acc)
+
     # L2 norm grad clip
     # https://github.com/NVIDIA/NeMo/blob/3368c3fc0b4a186ab33a1d68a504315100c0b2a6/nemo/collections/nlp/modules/common/megatron/clip_grads.py#L57
     # https://docs.pytorch.org/docs/stable/generated/torch.nn.utils.clip_grad_norm_.html
     if not getenv("DISABLE_GRAD_CLIP_NORM"):
       total_norm = Tensor(0.0, dtype=dtypes.float32, device=optim.params[0].device)
-      for p in optim.params:
-        total_norm += p.grad.float().square().sum()
-      total_norm = total_norm.sqrt().contiguous()
-      for p in optim.params:
-        p.grad = p.grad * (opt_gradient_clip_norm / (total_norm + 1e-6)).clamp(max_=1.0)
+      for g in grads:
+        total_norm += g.float().square().sum()
+      total_norm = total_norm.sqrt().contiguous().realize()
+      for g in grads:
+        g.assign((g * (opt_gradient_clip_norm / (total_norm + 1e-6)).clamp(max_=1.0)).cast(g.dtype)).realize()
 
     optim.step()
     scheduler.step()
 
+    for g in grads:
+      g.assign(g.zeros_like().contiguous()).realize()
+
     lr = optim.lr
-    loss.realize(lr)
-    return loss, lr
+    Tensor.realize(lr, *grads)
+
+    return lr
 
   @TinyJit
   @Tensor.train(False)
-  def eval_step(model, tokens:Tensor):
+  def eval_step(tokens:Tensor):
     if (DP := getenv("DP", 1)) > 1:
       device = tuple(f"{Device.DEFAULT}:{i}" for i in range(DP))
       tokens = tokens.shard(device, 0)
@@ -1432,39 +1459,73 @@ def train_llama3():
     eval_dataset = None
   else:
     from examples.mlperf.dataloader import get_llama3_dataset
-    eval_dataset = get_llama3_dataset(5760, SEQLEN, BASEDIR, val=True, small=bool(SMALL))
+    eval_dataset = get_llama3_dataset(EVAL_SAMPLES, SEQLEN, BASEDIR, val=True, small=bool(SMALL))
 
   def get_eval_iter():
     if eval_dataset is None:
-      return fake_data(EVAL_BS, 5760)
+      return fake_data(EVAL_BS, EVAL_SAMPLES)
     from examples.mlperf.dataloader import iterate_llama3_dataset
     return iterate_llama3_dataset(eval_dataset, EVAL_BS)
 
-  iter = get_train_iter()
+  num_params = sum(p.numel() for p in params) - model_params["vocab_size"]*model_params["dim"]
+  train_iter = get_train_iter()
   i, sequences_seen = resume_ckpt, 0
-  for tokens in tqdm(iter, total=SAMPLES//GBS):
+  step_times = []
+  while i < MAX_STEPS:
     GlobalCounters.reset()
     if getenv("TRAIN", 1):
-      t = time.perf_counter()
-      loss, lr = train_step(model, tokens)
+      profile_marker(f"train @ {i}")
+      st = time.perf_counter()
+
+      stopped = False
+      for _ in range(grad_acc):
+        ist = time.perf_counter()
+        try: tokens = next(train_iter)
+        except StopIteration:
+          stopped = True
+          break
+        dt = time.perf_counter()
+        loss = minibatch(tokens)
+      if stopped: break
+
+      gt = time.perf_counter()
+      lr = optim_step()
+      ot = time.perf_counter()
+
       loss = loss.float().item()
       lr = lr.item()
 
+      et = time.perf_counter()
+      step_time = et - st
+      gbs_time = gt - st
+      optim_time = ot - gt
+      data_time = dt - ist
+      dev_time = step_time - data_time * grad_acc
+      if BENCHMARK: step_times.append(step_time)
+
       i += 1
-      sequences_seen += tokens.shape[0]
+      sequences_seen += GBS
 
-      sec = time.perf_counter()-t
       mem_gb = GlobalCounters.mem_used / 1e9
-      gflops = GlobalCounters.global_ops / 1e9 / sec
+      gflops = GlobalCounters.global_ops / 1e9 / dev_time
+      mfu = ((6 * num_params * SEQLEN * GBS) / (dev_time * max(getenv("DP", 1), getenv("MP", 1)) * 2.3e15)) * 100
       tqdm.write(
-        f"{i:5} {sec:.2f} s run, {loss:.4f} loss, {lr:.12f} LR, {mem_gb:.2f} GB used, {gflops:9.2f} GFLOPS")
-
-      if (fname:=getenv("LOSS_FILE", "")):
-        with open(fname, "a") as f:
-          f.write(f"{i} {loss:.4f} {lr:.12f} {mem_gb:.2f}\n")
+          f"{i:5} {step_time:.3f} s step, {gbs_time:.3f} s gbs, {optim_time:.3f} s optim, {data_time:.3f} s data, {loss:.4f} loss, " \
+          f"{lr:.12f} LR, {mem_gb:.2f} GB used, {gflops:9.2f} GFLOPS, {mfu:5.2f}% MFU")
 
       if WANDB:
-        wandb.log({"lr": lr, "train/loss": loss, "train/step_time": sec, "train/GFLOPS": gflops, "train/sequences_seen": sequences_seen})
+        wandb.log({
+          "lr": lr, "train/loss": loss,
+          "train/step_time": step_time,
+          "train/gbs_time": gbs_time,
+          "train/optim_time": optim_time,
+          "train/dev_time": dev_time,
+          "train/data_time": data_time,
+          "train/mem": mem_gb,
+          "train/GFLOPS": gflops,
+          "train/MFU": mfu,
+          "train/sequences_seen": sequences_seen
+        })
 
       if (ckpt_freq := getenv("CKPT")) and (i % ckpt_freq == 0 and (i != 1 or ckpt_freq == 1)):
         tqdm.write("saving checkpoint")
@@ -1476,16 +1537,29 @@ def train_llama3():
         fn = f"{ckpt_dir}/llama3_{i}_optim.safe"
         safe_save(get_state_dict(scheduler), fn)
 
-    if sequences_seen % EVAL_FREQ == 0 and (i != 1 or EVAL_FREQ == 1):
+      if i == BENCHMARK:
+        median_step_time = sorted(step_times)[(BENCHMARK + 1) // 2]
+        estimated_total_minutes = int(median_step_time * (SAMPLES // GBS) / 60)
+        print(f"Estimated training time: {estimated_total_minutes // 60}h{estimated_total_minutes % 60}m")
+        print(f"epoch global_ops: {GlobalCounters.global_ops:_}, "
+              f"epoch global_mem: {GlobalCounters.global_mem:_}")
+
+    if (sequences_seen % EVAL_FREQ == 0 and (i != 1 or EVAL_FREQ == 1)) or (BENCHMARK and i == BENCHMARK):
+      if EVAL_BS == 0: return
       tqdm.write(f"evaluating after {sequences_seen} sequences")
+      profile_marker(f"eval @ {i}")
 
       # run eval
       eval_losses = []
       eval_iter = get_eval_iter()
       tqdm.write(f"evaluating {5760//EVAL_BS} batches of {EVAL_BS} sequences")
 
-      for tokens in tqdm(eval_iter, total=5760//EVAL_BS):
-        eval_losses += eval_step(model, tokens).tolist()
+      for j,tokens in tqdm(enumerate(eval_iter), total=EVAL_SAMPLES//EVAL_BS):
+        eval_losses += eval_step(tokens).tolist()
+
+        if BENCHMARK and (j+1) == min(BENCHMARK, EVAL_SAMPLES//EVAL_BS):
+          return
+
       log_perplexity = Tensor(eval_losses).mean().float().item()
 
       tqdm.write(f"eval log perplexity: {log_perplexity:.4f}")
@@ -1571,7 +1645,7 @@ def train_stable_diffusion():
     loss, out_lr = loss.detach().to("CPU"), optimizer.lr.to("CPU")
     Tensor.realize(loss, out_lr)
     return loss, out_lr
-    
+
   # checkpointing takes ~9 minutes without this, and ~1 minute with this
   @TinyJit
   def ckpt_to_cpu():
@@ -1610,7 +1684,7 @@ def train_stable_diffusion():
     if i == 3:
       for _ in range(3): ckpt_to_cpu() # do this at the beginning of run to prevent OOM surprises when checkpointing
       print("BEAM COMPLETE", flush=True) # allows wrapper script to detect BEAM search completion and retry if it failed
-      
+
     total_train_time = time.perf_counter() - train_start_time
     if WANDB:
       wandb.log({"train/loss": loss_item, "train/lr": lr_item, "train/loop_time_prev": loop_time, "train/dl_time": dl_time, "train/step": i,

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_1xMI300X/dev_beam.sh

@@ -4,7 +4,7 @@ export PYTHONPATH="." AMD=1
 export MODEL="bert"
 export DEFAULT_FLOAT="HALF" GPUS=1 BS=128 EVAL_BS=128
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 
 export BEAM=3 BEAM_UOPS_MAX=4000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
 export IGNORE_JIT_FIRST_BEAM=1

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/dev_beam.sh

@@ -5,7 +5,7 @@ export MODEL="bert"
 export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/dev_run.sh

@@ -8,7 +8,7 @@ export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 export TRAIN_STEPS=3900
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI300X/run_and_time.sh

@@ -11,7 +11,7 @@ export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 export TRAIN_STEPS=3900
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/dev_run_fa.sh

@@ -8,7 +8,7 @@ export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 export TRAIN_STEPS=3900
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=5000000
 
 export BEAM=0 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/dev_run_fp8.sh

@@ -8,7 +8,7 @@ export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 export TRAIN_STEPS=3900
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=5000000
 
 export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_8xMI350X/run_and_time.sh

@@ -11,7 +11,7 @@ export DEFAULT_FLOAT="HALF" GPUS=8 BS=1024 EVAL_BS=1024
 export OPT_BASE_LEARNING_RATE=0.0011 OPT_LAMB_BETA_1=0.60466 OPT_LAMB_BETA_2=0.85437 DECAY=0.1
 export TRAIN_STEPS=3900
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=5000000
 
 export BEAM=3 BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/dev_beam.sh

@@ -4,7 +4,7 @@ export PYTHONPATH="." NV=1
 export MODEL="bert"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/dev_run.sh

@@ -4,7 +4,7 @@ export PYTHONPATH="." NV=1
 export MODEL="bert"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_green/run_and_time.sh

@@ -7,7 +7,7 @@ export MODEL="bert"
 export SUBMISSION_PLATFORM="tinybox_green"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=72 EVAL_BS=72
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=8 BEAM_UOPS_MAX=10000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_beam.sh

@@ -4,7 +4,7 @@ export PYTHONPATH="." AMD=1
 export MODEL="bert"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/dev_run.sh

@@ -4,7 +4,7 @@ export PYTHONPATH="." AMD=1
 export MODEL="bert"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/bert/implementations/tinybox_red/run_and_time.sh

@@ -7,7 +7,7 @@ export MODEL="bert"
 export SUBMISSION_PLATFORM="tinybox_red"
 export DEFAULT_FLOAT="HALF" SUM_DTYPE="HALF" GPUS=6 BS=96 EVAL_BS=96
 
-export IGNORE_OOB=1
+export CHECK_OOB=0
 export REWRITE_STACK_LIMIT=500000
 
 export BEAM=5 BEAM_UOPS_MAX=8000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_beam.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+export PYTHONPATH="."
+export DEV=${DEV:-AMD}
+export CHECK_OOB=0
+export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
+
+export DEBUG=${DEBUG:-2}
+export FLASH_ATTENTION=${FLASH_ATTENTION:-1}
+export ALL2ALL=${ALL2ALL:-1}
+
+export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
+export DP=8 BS=8 EVAL_BS=8 GRADIENT_ACC_STEPS=2
+export GBS=$((BS * GRADIENT_ACC_STEPS))
+
+export MODEL="llama3"
+export BASEDIR="/raid/datasets/c4-8b/"
+export SMALL=1
+export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"}
+export EVAL_TARGET=3.3 EVAL_FREQ=12288
+export LR="2.5e-4" END_LR="2.5e-5" WARMUP_SAMPLES=256 MAX_STEPS=1200000
+export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS))
+export SAMPLES=$((MAX_STEPS * GBS))
+
+export SEED=5760
+
+export JITBEAM=3
+export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
+
+export FAKEDATA=1 BENCHMARK=10 LLAMA_LAYERS=2
+
+python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+export PYTHONPATH="."
+export DEV=${DEV:-AMD}
+export EMULATE="AMD_CDNA4"
+export CHECK_OOB=0
+export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000
+
+export DEBUG=${DEBUG:-0}
+export FLASH_ATTENTION=${FLASH_ATTENTION:-1}
+export ALL2ALL=${ALL2ALL:-1}
+export USE_ATOMICS=${USE_ATOMICS:-1}
+export ASM_GEMM=${ASM_GEMM:-1}
+
+export DEFAULT_FLOAT="bfloat16" OPTIM_DTYPE="bfloat16"
+export DP=${DP:-8} BS=${BS:-8} EVAL_BS=${EVAL_BS:-8} GRADIENT_ACC_STEPS=${GRADIENT_ACC_STEPS:-2}
+export GBS=$((BS * GRADIENT_ACC_STEPS))
+
+export MODEL="llama3"
+export BASEDIR="/raid/datasets/c4-8b/"
+export SMALL=1
+export LLAMA3_SIZE=${LLAMA3_SIZE:-"8B"}
+export EVAL_TARGET=3.3 EVAL_FREQ=12288
+export LR="2.5e-4" END_LR="2.5e-5" WARMUP_SAMPLES=256 MAX_STEPS=1200000
+export WARMUP_STEPS=$((WARMUP_SAMPLES / GBS))
+export SAMPLES=$((MAX_STEPS * GBS))
+
+export SEED=${SEED:-5760}
+
+export JITBEAM=${JITBEAM:-3}
+export BEAM_UOPS_MAX=6000 BEAM_UPCAST_MAX=256 BEAM_LOCAL_MAX=1024 BEAM_MIN_PROGRESS=5
+
+python3 examples/mlperf/model_train.py

examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/startup_walltime.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+export BENCHMARK=5
+export EVAL_BS=0
+export FAKEDATA=1
+export HIP_VISIBLE_DEVICES=""
+export DEV=NULL
+export JITBEAM=0
+export LLAMA_LAYERS=${LLAMA_LAYERS:-"2"}
+time examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama8b/implementations/tinybox_8xMI350X/dev_run.sh

examples/webgpu/stable_diffusion/compile.py

@@ -93,7 +93,7 @@ if __name__ == "__main__":
     forward: Any = None
 
   sub_steps = [
-    Step(name = "textModel", input = [Tensor.randn(1, 77)], forward = model.cond_stage_model.transformer.text_model),
+    Step(name = "textModel", input = [Tensor.randint(1, 77, low=0, high=49408, dtype=dtypes.int32)], forward = model.cond_stage_model.transformer.text_model),
     Step(name = "diffusor", input = [Tensor.randn(1, 77, 768), Tensor.randn(1, 77, 768), Tensor.randn(1,4,64,64), Tensor.rand(1), Tensor.randn(1), Tensor.randn(1), Tensor.randn(1)], forward = model),
     Step(name = "decoder", input = [Tensor.randn(1,4,64,64)], forward = model.decode),
     Step(name = "f16tof32", input = [Tensor.randn(2097120, dtype=dtypes.uint32)], forward = u32_to_f16)

examples/whisper.py

@@ -7,6 +7,7 @@ from tinygrad import Tensor, TinyJit, Variable, nn, dtypes
 from tinygrad.nn.state import torch_load, load_state_dict
 from tinygrad.helpers import getenv, fetch
 
+from examples.audio_helpers import mel
 import numpy as np
 import librosa
 
@@ -159,7 +160,7 @@ def prep_audio(waveforms: List[np.ndarray], batch_size: int, truncate=False) ->
 
   stft = librosa.stft(waveforms, n_fft=N_FFT, hop_length=HOP_LENGTH, window='hann', dtype=np.csingle)
   magnitudes = np.absolute(stft[..., :-1]) ** 2
-  mel_spec = librosa.filters.mel(sr=RATE, n_fft=N_FFT, n_mels=N_MELS) @ magnitudes
+  mel_spec = mel(sr=RATE, n_fft=N_FFT, n_mels=N_MELS).numpy() @ magnitudes
 
   log_spec = np.log10(np.clip(mel_spec, 1e-10, None))
   log_spec = np.maximum(log_spec, log_spec.max((1,2), keepdims=True) - 8.0)

extra/assembly/amd/__init__.py

@@ -0,0 +1,67 @@
+# Instruction format detection and decoding
+from __future__ import annotations
+from extra.assembly.amd.dsl import Inst, FixedBitField, EnumBitField
+
+# SDWA/DPP variant detection: src0 field (bits 0-8) encodes the variant
+# 0xf9 (249) = SDWA, 0xfa (250) = DPP16 for CDNA (GFX9)
+_VARIANT_SRC0 = {"_SDWA_SDST": 0xf9, "_SDWA": 0xf9, "_DPP16": 0xfa}
+
+def _matches(data: bytes, cls: type[Inst]) -> bool:
+  """Check if data matches all FixedBitFields and op is in allowed."""
+  for _, field in cls._fields:
+    dword_idx = field.lo // 32
+    if len(data) < (dword_idx + 1) * 4: return False
+    word = int.from_bytes(data[dword_idx*4:(dword_idx+1)*4], 'little')
+    field_lo = field.lo % 32
+    if isinstance(field, FixedBitField):
+      if ((word >> field_lo) & field.mask) != field.default: return False
+    if isinstance(field, EnumBitField) and field.allowed is not None:
+      try: opcode = field.decode((word >> field_lo) & field.mask)
+      except ValueError: return False  # opcode not in enum
+      if opcode not in field.allowed: return False
+  # Check SDWA/DPP variant based on src0 field (bits 0-8) - only for variant classes
+  name = cls.__name__
+  word = int.from_bytes(data[:4], 'little')
+  for suffix, expected_src0 in _VARIANT_SRC0.items():
+    if name.endswith(suffix): return (word & 0x1ff) == expected_src0
+  return True
+
+# Import instruction classes for each architecture
+from extra.assembly.amd.autogen.rdna3.ins import (VOP1, VOP1_SDST, VOP1_LIT, VOP2, VOP2_LIT, VOP3, VOP3_SDST, VOP3SD, VOP3P, VOPC, VOPD, VINTERP,
+  SOP1, SOP1_LIT, SOP2, SOP2_LIT, SOPC, SOPK, SOPK_LIT, SOPP, SMEM, DS, FLAT, GLOBAL, SCRATCH)
+from extra.assembly.amd.autogen.rdna4.ins import (VOP1 as R4_VOP1, VOP1_SDST as R4_VOP1_SDST, VOP1_LIT as R4_VOP1_LIT,
+  VOP2 as R4_VOP2, VOP2_LIT as R4_VOP2_LIT, VOP3 as R4_VOP3, VOP3_SDST as R4_VOP3_SDST, VOP3SD as R4_VOP3SD, VOP3P as R4_VOP3P,
+  VOPC as R4_VOPC, VOPD as R4_VOPD, VINTERP as R4_VINTERP, SOP1 as R4_SOP1, SOP1_LIT as R4_SOP1_LIT,
+  SOP2 as R4_SOP2, SOP2_LIT as R4_SOP2_LIT, SOPC as R4_SOPC, SOPC_LIT as R4_SOPC_LIT,
+  SOPK as R4_SOPK, SOPK_LIT as R4_SOPK_LIT, SOPP as R4_SOPP,
+  SMEM as R4_SMEM, DS as R4_DS, VFLAT as R4_FLAT, VGLOBAL as R4_GLOBAL, VSCRATCH as R4_SCRATCH)
+from extra.assembly.amd.autogen.cdna.ins import (VOP1 as C_VOP1, VOP1_SDWA as C_VOP1_SDWA, VOP1_DPP16 as C_VOP1_DPP16,
+  VOP2 as C_VOP2, VOP2_LIT as C_VOP2_LIT, VOP2_SDWA as C_VOP2_SDWA, VOP2_DPP16 as C_VOP2_DPP16,
+  VOPC as C_VOPC, VOPC_SDWA_SDST as C_VOPC_SDWA_SDST,
+  VOP3 as C_VOP3, VOP3_SDST as C_VOP3_SDST, VOP3SD as C_VOP3SD, VOP3P as C_VOP3P, VOP3P_MFMA as C_VOP3P_MFMA, VOP3PX2 as C_VOP3PX2,
+  SOP1 as C_SOP1, SOP2 as C_SOP2, SOPC as C_SOPC, SOPK as C_SOPK, SOPK_LIT as C_SOPK_LIT, SOPP as C_SOPP, SMEM as C_SMEM, DS as C_DS,
+  FLAT as C_FLAT, GLOBAL as C_GLOBAL, SCRATCH as C_SCRATCH, MUBUF as C_MUBUF)
+
+# Order matters: more specific encodings first, catch-alls (SOP2, VOP2) last
+# Order: base before _LIT (base matches regular ops, _LIT catches lit-only ops excluded from base)
+_FORMATS = {
+  "rdna3": [VOPD, VOP3P, VINTERP, VOP3SD, VOP3_SDST, VOP3, DS, GLOBAL, SCRATCH, FLAT, SMEM,
+            SOP1, SOP1_LIT, SOP2, SOP2_LIT, SOPC, SOPK, SOPK_LIT, SOPP, VOPC, VOP1_SDST, VOP1, VOP1_LIT, VOP2, VOP2_LIT],
+  "rdna4": [R4_VOPD, R4_VOP3P, R4_VINTERP, R4_VOP3SD, R4_VOP3_SDST, R4_VOP3, R4_DS, R4_GLOBAL, R4_SCRATCH, R4_FLAT, R4_SMEM,
+            R4_SOP1, R4_SOP1_LIT, R4_SOPC, R4_SOPC_LIT, R4_SOPP, R4_SOPK, R4_SOPK_LIT, R4_VOPC, R4_VOP1_SDST, R4_VOP1, R4_VOP1_LIT,
+            R4_SOP2, R4_SOP2_LIT, R4_VOP2, R4_VOP2_LIT],
+  "cdna": [C_VOP3PX2, C_VOP3P_MFMA, C_VOP3P, C_VOP3SD, C_VOP3_SDST, C_VOP3, C_DS, C_GLOBAL, C_SCRATCH, C_FLAT, C_MUBUF, C_SMEM,
+            C_SOP1, C_SOPC, C_SOPP, C_SOPK, C_SOPK_LIT, C_VOPC_SDWA_SDST, C_VOPC,
+            C_VOP1_DPP16, C_VOP1_SDWA, C_VOP1, C_VOP2_DPP16, C_VOP2_SDWA, C_SOP2, C_VOP2, C_VOP2_LIT],
+}
+
+def detect_format(data: bytes, arch: str = "rdna3") -> type[Inst]:
+  """Detect instruction format from machine code bytes."""
+  assert len(data) >= 4, f"need at least 4 bytes, got {len(data)}"
+  for cls in _FORMATS[arch]:
+    if _matches(data, cls): return cls
+  raise ValueError(f"unknown {arch} format word={int.from_bytes(data[:4], 'little'):#010x}")
+
+def decode_inst(data: bytes, arch: str = "rdna3") -> Inst:
+  """Decode machine code bytes into an instruction."""
+  return detect_format(data, arch).from_bytes(data)

extra/assembly/amd/autogen/cdna/str_pcode.py

@@ -1,6 +1,6 @@
 # autogenerated from AMD ISA PDF - do not edit
 # ruff: noqa: E501
-from extra.assembly.amd.autogen.cdna.enum import DSOp, FLATOp, GLOBALOp, MTBUFOp, MUBUFOp, SCRATCHOp, SMEMOp, SOP1Op, SOP2Op, SOPCOp, SOPKOp, SOPPOp, VOP1Op, VOP2Op, VOP3AOp, VOP3BOp, VOP3POp, VOPCOp
+from extra.assembly.amd.autogen.cdna.enum import DSOp, FLATOp, GLOBALOp, MTBUFOp, MUBUFOp, SCRATCHOp, SMEMOp, SOP1Op, SOP2Op, SOPCOp, SOPKOp, SOPPOp, VOP1Op, VOP2Op, VOP3Op, VOP3POp, VOP3SDOp, VOPCOp
 
 PCODE = {
   DSOp.DS_ADD_U32: 'addr = CalcDsAddr(ADDR.b32, OFFSET0.b32, OFFSET1.b32);\ntmp = MEM[addr].u32;\nMEM[addr].u32 += DATA.u32;\nRETURN_DATA.u32 = tmp',
@@ -722,428 +722,418 @@ PCODE = {
   VOP2Op.V_FMAC_F32_E32: 'D0.f32 = fma(S0.f32, S1.f32, D0.f32)',
   VOP2Op.V_PK_FMAC_F16_E32: 'D0[15 : 0].f16 = fma(S0[15 : 0].f16, S1[15 : 0].f16, D0[15 : 0].f16);\nD0[31 : 16].f16 = fma(S0[31 : 16].f16, S1[31 : 16].f16, D0[31 : 16].f16)',
   VOP2Op.V_XNOR_B32_E32: 'D0.u32 = ~(S0.u32 ^ S1.u32)',
-  VOP3AOp.V_CMP_CLASS_F32: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_CLASS_F32: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
-  VOP3AOp.V_CMP_CLASS_F64: "declare result : 1'U;\nif isSignalNAN(S0.f64) then\nresult = S1.u32[0]\nelsif isQuietNAN(S0.f64) then\nresult = S1.u32[1]\nelsif exponent(S0.f64) == 2047 then\n// +-INF\nresult = S1.u32[sign(S0.f64) ? 2 : 9]\nelsif exponent(S0.f64) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f64) ? 3 : 8]\nelsif abs(S0.f64) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f64) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f64) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_CLASS_F64: "declare result : 1'U;\nif isSignalNAN(S0.f64) then\nresult = S1.u32[0]\nelsif isQuietNAN(S0.f64) then\nresult = S1.u32[1]\nelsif exponent(S0.f64) == 2047 then\n// +-INF\nresult = S1.u32[sign(S0.f64) ? 2 : 9]\nelsif exponent(S0.f64) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f64) ? 3 : 8]\nelsif abs(S0.f64) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f64) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f64) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
-  VOP3AOp.V_CMP_CLASS_F16: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f16)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f16)) then\nresult = S1.u32[1]\nelsif exponent(S0.f16) == 31 then\n// +-INF\nresult = S1.u32[sign(S0.f16) ? 2 : 9]\nelsif exponent(S0.f16) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f16) ? 3 : 8]\nelsif 64'F(abs(S0.f16)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f16) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f16) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_CLASS_F16: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f16)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f16)) then\nresult = S1.u32[1]\nelsif exponent(S0.f16) == 31 then\n// +-INF\nresult = S1.u32[sign(S0.f16) ? 2 : 9]\nelsif exponent(S0.f16) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f16) ? 3 : 8]\nelsif 64'F(abs(S0.f16)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f16) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f16) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
-  VOP3AOp.V_CMP_F_F16: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_F16: 'D0.u64[laneId] = S0.f16 < S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_F16: 'D0.u64[laneId] = S0.f16 == S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_F16: 'D0.u64[laneId] = S0.f16 <= S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_F16: 'D0.u64[laneId] = S0.f16 > S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LG_F16: 'D0.u64[laneId] = S0.f16 <> S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_F16: 'D0.u64[laneId] = S0.f16 >= S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_O_F16: "D0.u64[laneId] = (!isNAN(64'F(S0.f16)) && !isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_U_F16: "D0.u64[laneId] = (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_NGE_F16: 'D0.u64[laneId] = !(S0.f16 >= S1.f16);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLG_F16: 'D0.u64[laneId] = !(S0.f16 <> S1.f16);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NGT_F16: 'D0.u64[laneId] = !(S0.f16 > S1.f16);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLE_F16: 'D0.u64[laneId] = !(S0.f16 <= S1.f16);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NEQ_F16: 'D0.u64[laneId] = !(S0.f16 == S1.f16);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLT_F16: 'D0.u64[laneId] = !(S0.f16 < S1.f16);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_TRU_F16: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_F16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 < S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 == S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 <= S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 > S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LG_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 <> S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 >= S1.f16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_O_F16: "EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(64'F(S0.f16)) && !isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_U_F16: "EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_NGE_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 >= S1.f16);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLG_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 <> S1.f16);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NGT_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 > S1.f16);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLE_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 <= S1.f16);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NEQ_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 == S1.f16);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLT_F16: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 < S1.f16);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_TRU_F16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_F32: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_F32: 'D0.u64[laneId] = S0.f32 < S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_F32: 'D0.u64[laneId] = S0.f32 == S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_F32: 'D0.u64[laneId] = S0.f32 <= S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_F32: 'D0.u64[laneId] = S0.f32 > S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LG_F32: 'D0.u64[laneId] = S0.f32 <> S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_F32: 'D0.u64[laneId] = S0.f32 >= S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_O_F32: "D0.u64[laneId] = (!isNAN(64'F(S0.f32)) && !isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_U_F32: "D0.u64[laneId] = (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_NGE_F32: 'D0.u64[laneId] = !(S0.f32 >= S1.f32);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLG_F32: 'D0.u64[laneId] = !(S0.f32 <> S1.f32);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NGT_F32: 'D0.u64[laneId] = !(S0.f32 > S1.f32);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLE_F32: 'D0.u64[laneId] = !(S0.f32 <= S1.f32);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NEQ_F32: 'D0.u64[laneId] = !(S0.f32 == S1.f32);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLT_F32: 'D0.u64[laneId] = !(S0.f32 < S1.f32);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_TRU_F32: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_F32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 < S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 == S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 <= S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 > S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LG_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 <> S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 >= S1.f32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_O_F32: "EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(64'F(S0.f32)) && !isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_U_F32: "EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_NGE_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 >= S1.f32);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLG_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 <> S1.f32);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NGT_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 > S1.f32);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLE_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 <= S1.f32);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NEQ_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 == S1.f32);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLT_F32: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 < S1.f32);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_TRU_F32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_F64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_F64: 'D0.u64[laneId] = S0.f64 < S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_F64: 'D0.u64[laneId] = S0.f64 == S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_F64: 'D0.u64[laneId] = S0.f64 <= S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_F64: 'D0.u64[laneId] = S0.f64 > S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LG_F64: 'D0.u64[laneId] = S0.f64 <> S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_F64: 'D0.u64[laneId] = S0.f64 >= S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_O_F64: 'D0.u64[laneId] = (!isNAN(S0.f64) && !isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_U_F64: 'D0.u64[laneId] = (isNAN(S0.f64) || isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NGE_F64: 'D0.u64[laneId] = !(S0.f64 >= S1.f64);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLG_F64: 'D0.u64[laneId] = !(S0.f64 <> S1.f64);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NGT_F64: 'D0.u64[laneId] = !(S0.f64 > S1.f64);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLE_F64: 'D0.u64[laneId] = !(S0.f64 <= S1.f64);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NEQ_F64: 'D0.u64[laneId] = !(S0.f64 == S1.f64);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NLT_F64: 'D0.u64[laneId] = !(S0.f64 < S1.f64);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_TRU_F64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_F64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 < S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 == S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 <= S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 > S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LG_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 <> S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 >= S1.f64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_O_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(S0.f64) && !isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_U_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(S0.f64) || isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NGE_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 >= S1.f64);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLG_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 <> S1.f64);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NGT_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 > S1.f64);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLE_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 <= S1.f64);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NEQ_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 == S1.f64);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NLT_F64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 < S1.f64);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_TRU_F64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_I16: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_I16: 'D0.u64[laneId] = S0.i16 < S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_I16: 'D0.u64[laneId] = S0.i16 == S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_I16: 'D0.u64[laneId] = S0.i16 <= S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_I16: 'D0.u64[laneId] = S0.i16 > S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_I16: 'D0.u64[laneId] = S0.i16 <> S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_I16: 'D0.u64[laneId] = S0.i16 >= S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_I16: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_U16: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_U16: 'D0.u64[laneId] = S0.u16 < S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_U16: 'D0.u64[laneId] = S0.u16 == S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_U16: 'D0.u64[laneId] = S0.u16 <= S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_U16: 'D0.u64[laneId] = S0.u16 > S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_U16: 'D0.u64[laneId] = S0.u16 <> S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_U16: 'D0.u64[laneId] = S0.u16 >= S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_U16: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_I16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 < S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 == S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 <= S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 > S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 <> S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_I16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 >= S1.i16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_I16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_U16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 < S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 == S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 <= S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 > S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 <> S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_U16: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 >= S1.u16;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_U16: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_I32: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_I32: 'D0.u64[laneId] = S0.i32 < S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_I32: 'D0.u64[laneId] = S0.i32 == S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_I32: 'D0.u64[laneId] = S0.i32 <= S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_I32: 'D0.u64[laneId] = S0.i32 > S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_I32: 'D0.u64[laneId] = S0.i32 <> S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_I32: 'D0.u64[laneId] = S0.i32 >= S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_I32: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_U32: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_U32: 'D0.u64[laneId] = S0.u32 < S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_U32: 'D0.u64[laneId] = S0.u32 == S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_U32: 'D0.u64[laneId] = S0.u32 <= S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_U32: 'D0.u64[laneId] = S0.u32 > S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_U32: 'D0.u64[laneId] = S0.u32 <> S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_U32: 'D0.u64[laneId] = S0.u32 >= S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_U32: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_I32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 < S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 == S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 <= S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 > S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 <> S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_I32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 >= S1.i32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_I32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_U32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 < S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 == S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 <= S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 > S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 <> S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_U32: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 >= S1.u32;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_U32: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_I64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_I64: 'D0.u64[laneId] = S0.i64 < S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_I64: 'D0.u64[laneId] = S0.i64 == S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_I64: 'D0.u64[laneId] = S0.i64 <= S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_I64: 'D0.u64[laneId] = S0.i64 > S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_I64: 'D0.u64[laneId] = S0.i64 <> S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_I64: 'D0.u64[laneId] = S0.i64 >= S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_I64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_F_U64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMP_LT_U64: 'D0.u64[laneId] = S0.u64 < S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_EQ_U64: 'D0.u64[laneId] = S0.u64 == S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_LE_U64: 'D0.u64[laneId] = S0.u64 <= S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GT_U64: 'D0.u64[laneId] = S0.u64 > S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_NE_U64: 'D0.u64[laneId] = S0.u64 <> S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_GE_U64: 'D0.u64[laneId] = S0.u64 >= S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMP_T_U64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_I64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 < S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 == S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 <= S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 > S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 <> S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_I64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 >= S1.i64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_I64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_F_U64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CMPX_LT_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 < S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_EQ_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 == S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_LE_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 <= S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GT_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 > S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_NE_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 <> S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_GE_U64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 >= S1.u64;\n// D0 = VCC in VOPC encoding.',
-  VOP3AOp.V_CMPX_T_U64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
-  VOP3AOp.V_CNDMASK_B32: 'D0.u32 = VCC.u64[laneId] ? S1.u32 : S0.u32',
-  VOP3AOp.V_ADD_F32: 'D0.f32 = S0.f32 + S1.f32',
-  VOP3AOp.V_SUB_F32: 'D0.f32 = S0.f32 - S1.f32',
-  VOP3AOp.V_SUBREV_F32: 'D0.f32 = S1.f32 - S0.f32',
-  VOP3AOp.V_FMAC_F64: 'D0.f64 = fma(S0.f64, S1.f64, D0.f64)',
-  VOP3AOp.V_MUL_F32: 'D0.f32 = S0.f32 * S1.f32',
-  VOP3AOp.V_MUL_I32_I24: "D0.i32 = 32'I(S0.i24) * 32'I(S1.i24)",
-  VOP3AOp.V_MUL_HI_I32_I24: "D0.i32 = 32'I((64'I(S0.i24) * 64'I(S1.i24)) >> 32U)",
-  VOP3AOp.V_MUL_U32_U24: "D0.u32 = 32'U(S0.u24) * 32'U(S1.u24)",
-  VOP3AOp.V_MUL_HI_U32_U24: "D0.u32 = 32'U((64'U(S0.u24) * 64'U(S1.u24)) >> 32U)",
-  VOP3AOp.V_MIN_F32: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S0.f32)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif isNAN(64'F(S0.f32)) then\nD0.f32 = S1.f32\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == +0.0) && (64'F(S1.f32) == -0.0)) then\nD0.f32 = S1.f32\nelsif ((64'F(S0.f32) == -0.0) && (64'F(S1.f32) == +0.0)) then\nD0.f32 = S0.f32\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F32.\nD0.f32 = S0.f32 < S1.f32 ? S0.f32 : S1.f32\nendif",
-  VOP3AOp.V_MAX_F32: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S0.f32)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif isNAN(64'F(S0.f32)) then\nD0.f32 = S1.f32\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == +0.0) && (64'F(S1.f32) == -0.0)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == -0.0) && (64'F(S1.f32) == +0.0)) then\nD0.f32 = S1.f32\nelsif WAVE_MODE.IEEE then\nD0.f32 = S0.f32 >= S1.f32 ? S0.f32 : S1.f32\nelse\nD0.f32 = S0.f32 > S1.f32 ? S0.f32 : S1.f32\nendif",
-  VOP3AOp.V_MIN_I32: 'D0.i32 = S0.i32 < S1.i32 ? S0.i32 : S1.i32',
-  VOP3AOp.V_MAX_I32: 'D0.i32 = S0.i32 >= S1.i32 ? S0.i32 : S1.i32',
-  VOP3AOp.V_MIN_U32: 'D0.u32 = S0.u32 < S1.u32 ? S0.u32 : S1.u32',
-  VOP3AOp.V_MAX_U32: 'D0.u32 = S0.u32 >= S1.u32 ? S0.u32 : S1.u32',
-  VOP3AOp.V_LSHRREV_B32: 'D0.u32 = (S1.u32 >> S0[4 : 0].u32)',
-  VOP3AOp.V_ASHRREV_I32: 'D0.i32 = (S1.i32 >> S0[4 : 0].u32)',
-  VOP3AOp.V_LSHLREV_B32: 'D0.u32 = (S1.u32 << S0[4 : 0].u32)',
-  VOP3AOp.V_AND_B32: 'D0.u32 = (S0.u32 & S1.u32)',
-  VOP3AOp.V_OR_B32: 'D0.u32 = (S0.u32 | S1.u32)',
-  VOP3AOp.V_XOR_B32: 'D0.u32 = (S0.u32 ^ S1.u32)',
-  VOP3AOp.V_DOT2C_F32_BF16: 'tmp = D0.f32;\ntmp += bf16_to_f32(S0[15 : 0].bf16) * bf16_to_f32(S1[15 : 0].bf16);\ntmp += bf16_to_f32(S0[31 : 16].bf16) * bf16_to_f32(S1[31 : 16].bf16);\nD0.f32 = tmp',
-  VOP3AOp.V_ADD_F16: 'D0.f16 = S0.f16 + S1.f16',
-  VOP3AOp.V_SUB_F16: 'D0.f16 = S0.f16 - S1.f16',
-  VOP3AOp.V_SUBREV_F16: 'D0.f16 = S1.f16 - S0.f16',
-  VOP3AOp.V_MUL_F16: 'D0.f16 = S0.f16 * S1.f16',
-  VOP3AOp.V_MAC_F16: "tmp = S0.f16 * S1.f16 + D0.f16;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
-  VOP3AOp.V_ADD_U16: 'D0.u16 = S0.u16 + S1.u16',
-  VOP3AOp.V_SUB_U16: 'D0.u16 = S0.u16 - S1.u16',
-  VOP3AOp.V_SUBREV_U16: 'D0.u16 = S1.u16 - S0.u16',
-  VOP3AOp.V_MUL_LO_U16: 'D0.u16 = S0.u16 * S1.u16',
-  VOP3AOp.V_LSHLREV_B16: 'D0.u16 = (S1.u16 << S0[3 : 0].u32)',
-  VOP3AOp.V_LSHRREV_B16: 'D0.u16 = (S1.u16 >> S0[3 : 0].u32)',
-  VOP3AOp.V_ASHRREV_I16: 'D0.i16 = (S1.i16 >> S0[3 : 0].u32)',
-  VOP3AOp.V_MAX_F16: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == +0.0) && (64'F(S1.f16) == -0.0)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == -0.0) && (64'F(S1.f16) == +0.0)) then\nD0.f16 = S1.f16\nelsif WAVE_MODE.IEEE then\nD0.f16 = S0.f16 >= S1.f16 ? S0.f16 : S1.f16\nelse\nD0.f16 = S0.f16 > S1.f16 ? S0.f16 : S1.f16\nendif",
-  VOP3AOp.V_MIN_F16: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == +0.0) && (64'F(S1.f16) == -0.0)) then\nD0.f16 = S1.f16\nelsif ((64'F(S0.f16) == -0.0) && (64'F(S1.f16) == +0.0)) then\nD0.f16 = S0.f16\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F16.\nD0.f16 = S0.f16 < S1.f16 ? S0.f16 : S1.f16\nendif",
-  VOP3AOp.V_MAX_U16: 'D0.u16 = S0.u16 >= S1.u16 ? S0.u16 : S1.u16',
-  VOP3AOp.V_MAX_I16: 'D0.i16 = S0.i16 >= S1.i16 ? S0.i16 : S1.i16',
-  VOP3AOp.V_MIN_U16: 'D0.u16 = S0.u16 < S1.u16 ? S0.u16 : S1.u16',
-  VOP3AOp.V_MIN_I16: 'D0.i16 = S0.i16 < S1.i16 ? S0.i16 : S1.i16',
-  VOP3AOp.V_LDEXP_F16: "D0.f16 = S0.f16 * 16'F(2.0F ** 32'I(S1.i16))",
-  VOP3AOp.V_ADD_U32: 'D0.u32 = S0.u32 + S1.u32',
-  VOP3AOp.V_SUB_U32: 'D0.u32 = S0.u32 - S1.u32',
-  VOP3AOp.V_SUBREV_U32: 'D0.u32 = S1.u32 - S0.u32',
-  VOP3AOp.V_DOT2C_F32_F16: 'tmp = D0.f32;\ntmp += f16_to_f32(S0[15 : 0].f16) * f16_to_f32(S1[15 : 0].f16);\ntmp += f16_to_f32(S0[31 : 16].f16) * f16_to_f32(S1[31 : 16].f16);\nD0.f32 = tmp',
-  VOP3AOp.V_DOT2C_I32_I16: 'tmp = D0.i32;\ntmp += i16_to_i32(S0[15 : 0].i16) * i16_to_i32(S1[15 : 0].i16);\ntmp += i16_to_i32(S0[31 : 16].i16) * i16_to_i32(S1[31 : 16].i16);\nD0.i32 = tmp',
-  VOP3AOp.V_DOT4C_I32_I8: 'tmp = D0.i32;\ntmp += i8_to_i32(S0[7 : 0].i8) * i8_to_i32(S1[7 : 0].i8);\ntmp += i8_to_i32(S0[15 : 8].i8) * i8_to_i32(S1[15 : 8].i8);\ntmp += i8_to_i32(S0[23 : 16].i8) * i8_to_i32(S1[23 : 16].i8);\ntmp += i8_to_i32(S0[31 : 24].i8) * i8_to_i32(S1[31 : 24].i8);\nD0.i32 = tmp',
-  VOP3AOp.V_DOT8C_I32_I4: 'tmp = D0.i32;\ntmp += i4_to_i32(S0[3 : 0].i4) * i4_to_i32(S1[3 : 0].i4);\ntmp += i4_to_i32(S0[7 : 4].i4) * i4_to_i32(S1[7 : 4].i4);\ntmp += i4_to_i32(S0[11 : 8].i4) * i4_to_i32(S1[11 : 8].i4);\ntmp += i4_to_i32(S0[15 : 12].i4) * i4_to_i32(S1[15 : 12].i4);\ntmp += i4_to_i32(S0[19 : 16].i4) * i4_to_i32(S1[19 : 16].i4);\ntmp += i4_to_i32(S0[23 : 20].i4) * i4_to_i32(S1[23 : 20].i4);\ntmp += i4_to_i32(S0[27 : 24].i4) * i4_to_i32(S1[27 : 24].i4);\ntmp += i4_to_i32(S0[31 : 28].i4) * i4_to_i32(S1[31 : 28].i4);\nD0.i32 = tmp',
-  VOP3AOp.V_FMAC_F32: 'D0.f32 = fma(S0.f32, S1.f32, D0.f32)',
-  VOP3AOp.V_PK_FMAC_F16: 'D0[15 : 0].f16 = fma(S0[15 : 0].f16, S1[15 : 0].f16, D0[15 : 0].f16);\nD0[31 : 16].f16 = fma(S0[31 : 16].f16, S1[31 : 16].f16, D0[31 : 16].f16)',
-  VOP3AOp.V_XNOR_B32: 'D0.u32 = ~(S0.u32 ^ S1.u32)',
-  VOP3AOp.V_MAD_I32_I24: "D0.i32 = 32'I(S0.i24) * 32'I(S1.i24) + S2.i32",
-  VOP3AOp.V_MAD_U32_U24: "D0.u32 = 32'U(S0.u24) * 32'U(S1.u24) + S2.u32",
-  VOP3AOp.V_CUBEID_F32: '// Set D0.f = cubemap face ID ({0.0, 1.0, ..., 5.0}).\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nif S2.f32 < 0.0F then\nD0.f32 = 5.0F\nelse\nD0.f32 = 4.0F\nendif\nelsif abs(S1.f32) >= abs(S0.f32) then\nif S1.f32 < 0.0F then\nD0.f32 = 3.0F\nelse\nD0.f32 = 2.0F\nendif\nelse\nif S0.f32 < 0.0F then\nD0.f32 = 1.0F\nelse\nD0.f32 = 0.0F\nendif\nendif',
-  VOP3AOp.V_CUBESC_F32: '// D0.f = cubemap S coordinate.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nif S2.f32 < 0.0F then\nD0.f32 = -S0.f32\nelse\nD0.f32 = S0.f32\nendif\nelsif abs(S1.f32) >= abs(S0.f32) then\nD0.f32 = S0.f32\nelse\nif S0.f32 < 0.0F then\nD0.f32 = S2.f32\nelse\nD0.f32 = -S2.f32\nendif\nendif',
-  VOP3AOp.V_CUBETC_F32: '// D0.f = cubemap T coordinate.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nD0.f32 = -S1.f32\nelsif abs(S1.f32) >= abs(S0.f32) then\nif S1.f32 < 0.0F then\nD0.f32 = -S2.f32\nelse\nD0.f32 = S2.f32\nendif\nelse\nD0.f32 = -S1.f32\nendif',
-  VOP3AOp.V_CUBEMA_F32: '// D0.f = 2.0 * cubemap major axis.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nD0.f32 = S2.f32 * 2.0F\nelsif abs(S1.f32) >= abs(S0.f32) then\nD0.f32 = S1.f32 * 2.0F\nelse\nD0.f32 = S0.f32 * 2.0F\nendif',
-  VOP3AOp.V_BFE_U32: 'D0.u32 = ((S0.u32 >> S1[4 : 0].u32) & ((1U << S2[4 : 0].u32) - 1U))',
-  VOP3AOp.V_BFE_I32: 'tmp.i32 = ((S0.i32 >> S1[4 : 0].u32) & ((1 << S2[4 : 0].u32) - 1));\nD0.i32 = signext_from_bit(tmp.i32, S2[4 : 0].u32)',
-  VOP3AOp.V_BFI_B32: 'D0.u32 = ((S0.u32 & S1.u32) | (~S0.u32 & S2.u32))',
-  VOP3AOp.V_FMA_F32: 'D0.f32 = fma(S0.f32, S1.f32, S2.f32)',
-  VOP3AOp.V_FMA_F64: 'D0.f64 = fma(S0.f64, S1.f64, S2.f64)',
-  VOP3AOp.V_LERP_U8: 'tmp = ((S0.u32[31 : 24] + S1.u32[31 : 24] + S2.u32[24].u8) >> 1U << 24U);\ntmp += ((S0.u32[23 : 16] + S1.u32[23 : 16] + S2.u32[16].u8) >> 1U << 16U);\ntmp += ((S0.u32[15 : 8] + S1.u32[15 : 8] + S2.u32[8].u8) >> 1U << 8U);\ntmp += ((S0.u32[7 : 0] + S1.u32[7 : 0] + S2.u32[0].u8) >> 1U);\nD0.u32 = tmp.u32',
-  VOP3AOp.V_ALIGNBIT_B32: "D0.u32 = 32'U(({ S0.u32, S1.u32 } >> S2.u32[4 : 0]) & 0xffffffffLL)",
-  VOP3AOp.V_ALIGNBYTE_B32: "D0.u32 = 32'U(({ S0.u32, S1.u32 } >> (S2.u32[1 : 0] * 8U)) & 0xffffffffLL)",
-  VOP3AOp.V_MIN3_F32: 'D0.f32 = v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)',
-  VOP3AOp.V_MIN3_I32: 'D0.i32 = v_min_i32(v_min_i32(S0.i32, S1.i32), S2.i32)',
-  VOP3AOp.V_MIN3_U32: 'D0.u32 = v_min_u32(v_min_u32(S0.u32, S1.u32), S2.u32)',
-  VOP3AOp.V_MAX3_F32: 'D0.f32 = v_max_f32(v_max_f32(S0.f32, S1.f32), S2.f32)',
-  VOP3AOp.V_MAX3_I32: 'D0.i32 = v_max_i32(v_max_i32(S0.i32, S1.i32), S2.i32)',
-  VOP3AOp.V_MAX3_U32: 'D0.u32 = v_max_u32(v_max_u32(S0.u32, S1.u32), S2.u32)',
-  VOP3AOp.V_MED3_F32: "if (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)) || isNAN(64'F(S2.f32))) then\nD0.f32 = v_min3_f32(S0.f32, S1.f32, S2.f32)\nelsif v_max3_f32(S0.f32, S1.f32, S2.f32) == S0.f32 then\nD0.f32 = v_max_f32(S1.f32, S2.f32)\nelsif v_max3_f32(S0.f32, S1.f32, S2.f32) == S1.f32 then\nD0.f32 = v_max_f32(S0.f32, S2.f32)\nelse\nD0.f32 = v_max_f32(S0.f32, S1.f32)\nendif",
-  VOP3AOp.V_MED3_I32: 'if v_max3_i32(S0.i32, S1.i32, S2.i32) == S0.i32 then\nD0.i32 = v_max_i32(S1.i32, S2.i32)\nelsif v_max3_i32(S0.i32, S1.i32, S2.i32) == S1.i32 then\nD0.i32 = v_max_i32(S0.i32, S2.i32)\nelse\nD0.i32 = v_max_i32(S0.i32, S1.i32)\nendif',
-  VOP3AOp.V_MED3_U32: 'if v_max3_u32(S0.u32, S1.u32, S2.u32) == S0.u32 then\nD0.u32 = v_max_u32(S1.u32, S2.u32)\nelsif v_max3_u32(S0.u32, S1.u32, S2.u32) == S1.u32 then\nD0.u32 = v_max_u32(S0.u32, S2.u32)\nelse\nD0.u32 = v_max_u32(S0.u32, S1.u32)\nendif',
-  VOP3AOp.V_SAD_U8: "ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += 32'U(ABSDIFF(S0.u32[7 : 0], S1.u32[7 : 0]));\ntmp += 32'U(ABSDIFF(S0.u32[15 : 8], S1.u32[15 : 8]));\ntmp += 32'U(ABSDIFF(S0.u32[23 : 16], S1.u32[23 : 16]));\ntmp += 32'U(ABSDIFF(S0.u32[31 : 24], S1.u32[31 : 24]));\nD0.u32 = tmp",
-  VOP3AOp.V_SAD_HI_U8: "D0.u32 = (32'U(v_sad_u8(S0, S1, 0U)) << 16U) + S2.u32",
-  VOP3AOp.V_SAD_U16: 'ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += ABSDIFF(S0[15 : 0].u16, S1[15 : 0].u16);\ntmp += ABSDIFF(S0[31 : 16].u16, S1[31 : 16].u16);\nD0.u32 = tmp',
-  VOP3AOp.V_SAD_U32: 'ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\nD0.u32 = ABSDIFF(S0.u32, S1.u32) + S2.u32',
-  VOP3AOp.V_CVT_PK_U8_F32: "tmp = (S2.u32 & 32'U(~(0xff << (S1.u32[1 : 0].u32 * 8U))));\ntmp = (tmp | ((32'U(f32_to_u8(S0.f32)) & 255U) << (S1.u32[1 : 0].u32 * 8U)));\nD0.u32 = tmp",
-  VOP3AOp.V_DIV_FIXUP_F32: "sign_out = (sign(S1.f32) ^ sign(S2.f32));\nif isNAN(64'F(S2.f32)) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S2.f32)))\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif ((64'F(S1.f32) == 0.0) && (64'F(S2.f32) == 0.0)) then\n// 0/0\nD0.f32 = 32'F(0xffc00000)\nelsif ((64'F(abs(S1.f32)) == +INF) && (64'F(abs(S2.f32)) == +INF)) then\n// inf/inf\nD0.f32 = 32'F(0xffc00000)\nelsif ((64'F(S1.f32) == 0.0) || (64'F(abs(S2.f32)) == +INF)) then\n// x/0, or inf/y\nD0.f32 = sign_out ? -INF.f32 : +INF.f32\nelsif ((64'F(abs(S1.f32)) == +INF) || (64'F(S2.f32) == 0.0)) then\n// x/inf, 0/y\nD0.f32 = sign_out ? -0.0F : 0.0F\nelsif exponent(S2.f32) - exponent(S1.f32) < -150 then\nD0.f32 = sign_out ? -UNDERFLOW_F32 : UNDERFLOW_F32\nelsif exponent(S1.f32) == 255 then\nD0.f32 = sign_out ? -OVERFLOW_F32 : OVERFLOW_F32\nelse\nD0.f32 = sign_out ? -abs(S0.f32) : abs(S0.f32)\nendif",
-  VOP3AOp.V_DIV_FIXUP_F64: "sign_out = (sign(S1.f64) ^ sign(S2.f64));\nif isNAN(S2.f64) then\nD0.f64 = cvtToQuietNAN(S2.f64)\nelsif isNAN(S1.f64) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif ((S1.f64 == 0.0) && (S2.f64 == 0.0)) then\n// 0/0\nD0.f64 = 64'F(0xfff8000000000000LL)\nelsif ((abs(S1.f64) == +INF) && (abs(S2.f64) == +INF)) then\n// inf/inf\nD0.f64 = 64'F(0xfff8000000000000LL)\nelsif ((S1.f64 == 0.0) || (abs(S2.f64) == +INF)) then\n// x/0, or inf/y\nD0.f64 = sign_out ? -INF : +INF\nelsif ((abs(S1.f64) == +INF) || (S2.f64 == 0.0)) then\n// x/inf, 0/y\nD0.f64 = sign_out ? -0.0 : 0.0\nelsif exponent(S2.f64) - exponent(S1.f64) < -1075 then\nD0.f64 = sign_out ? -UNDERFLOW_F64 : UNDERFLOW_F64\nelsif exponent(S1.f64) == 2047 then\nD0.f64 = sign_out ? -OVERFLOW_F64 : OVERFLOW_F64\nelse\nD0.f64 = sign_out ? -abs(S0.f64) : abs(S0.f64)\nendif",
-  VOP3AOp.V_DIV_FMAS_F32: 'if VCC.u64[laneId] then\nD0.f32 = 2.0F ** 32 * fma(S0.f32, S1.f32, S2.f32)\nelse\nD0.f32 = fma(S0.f32, S1.f32, S2.f32)\nendif',
-  VOP3AOp.V_DIV_FMAS_F64: 'if VCC.u64[laneId] then\nD0.f64 = 2.0 ** 64 * fma(S0.f64, S1.f64, S2.f64)\nelse\nD0.f64 = fma(S0.f64, S1.f64, S2.f64)\nendif',
-  VOP3AOp.V_MSAD_U8: "ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += S1.u32[7 : 0] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[7 : 0], S1.u32[7 : 0]));\ntmp += S1.u32[15 : 8] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[15 : 8], S1.u32[15 : 8]));\ntmp += S1.u32[23 : 16] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[23 : 16], S1.u32[23 : 16]));\ntmp += S1.u32[31 : 24] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[31 : 24], S1.u32[31 : 24]));\nD0.u32 = tmp",
-  VOP3AOp.V_QSAD_PK_U16_U8: "tmp[63 : 48] = 16'B(v_sad_u8(S0[55 : 24], S1[31 : 0], S2[63 : 48].u32));\ntmp[47 : 32] = 16'B(v_sad_u8(S0[47 : 16], S1[31 : 0], S2[47 : 32].u32));\ntmp[31 : 16] = 16'B(v_sad_u8(S0[39 : 8], S1[31 : 0], S2[31 : 16].u32));\ntmp[15 : 0] = 16'B(v_sad_u8(S0[31 : 0], S1[31 : 0], S2[15 : 0].u32));\nD0.b64 = tmp.b64",
-  VOP3AOp.V_MQSAD_PK_U16_U8: "tmp[63 : 48] = 16'B(v_msad_u8(S0[55 : 24], S1[31 : 0], S2[63 : 48].u32));\ntmp[47 : 32] = 16'B(v_msad_u8(S0[47 : 16], S1[31 : 0], S2[47 : 32].u32));\ntmp[31 : 16] = 16'B(v_msad_u8(S0[39 : 8], S1[31 : 0], S2[31 : 16].u32));\ntmp[15 : 0] = 16'B(v_msad_u8(S0[31 : 0], S1[31 : 0], S2[15 : 0].u32));\nD0.b64 = tmp.b64",
-  VOP3AOp.V_MQSAD_U32_U8: "tmp[127 : 96] = 32'B(v_msad_u8(S0[55 : 24], S1[31 : 0], S2[127 : 96].u32));\ntmp[95 : 64] = 32'B(v_msad_u8(S0[47 : 16], S1[31 : 0], S2[95 : 64].u32));\ntmp[63 : 32] = 32'B(v_msad_u8(S0[39 : 8], S1[31 : 0], S2[63 : 32].u32));\ntmp[31 : 0] = 32'B(v_msad_u8(S0[31 : 0], S1[31 : 0], S2[31 : 0].u32));\nD0.b128 = tmp.b128",
-  VOP3AOp.V_MAD_LEGACY_F16: "tmp = S0.f16 * S1.f16 + S2.f16;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
-  VOP3AOp.V_MAD_LEGACY_U16: "tmp = S0.u16 * S1.u16 + S2.u16;\nif OPSEL.u4[3] then\nD0 = { tmp.u16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.u16 }\nendif",
-  VOP3AOp.V_MAD_LEGACY_I16: "tmp = S0.i16 * S1.i16 + S2.i16;\nif OPSEL.u4[3] then\nD0 = { tmp.i16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.i16 }\nendif",
-  VOP3AOp.V_PERM_B32: "BYTE_PERMUTE = lambda(data, sel) (\ndeclare in : 8'B[8];\nfor i in 0 : 7 do\nin[i] = data[i * 8 + 7 : i * 8].b8\nendfor;\nif sel.u32 >= 13U then\nreturn 8'0xff\nelsif sel.u32 == 12U then\nreturn 8'0x0\nelsif sel.u32 == 11U then\nreturn in[7][7].b8 * 8'0xff\nelsif sel.u32 == 10U then\nreturn in[5][7].b8 * 8'0xff\nelsif sel.u32 == 9U then\nreturn in[3][7].b8 * 8'0xff\nelsif sel.u32 == 8U then\nreturn in[1][7].b8 * 8'0xff\nelse\nreturn in[sel]\nendif);\nD0[31 : 24] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[31 : 24]);\nD0[23 : 16] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[23 : 16]);\nD0[15 : 8] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[15 : 8]);\nD0[7 : 0] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[7 : 0])",
-  VOP3AOp.V_FMA_LEGACY_F16: "tmp = fma(S0.f16, S1.f16, S2.f16);\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
-  VOP3AOp.V_DIV_FIXUP_LEGACY_F16: "sign_out = (sign(S1.f16) ^ sign(S2.f16));\nif isNAN(64'F(S2.f16)) then\ntmp = cvtToQuietNAN(64'F(S2.f16))\nelsif isNAN(64'F(S1.f16)) then\ntmp = cvtToQuietNAN(64'F(S1.f16))\nelsif ((64'F(S1.f16) == 0.0) && (64'F(S2.f16) == 0.0)) then\n// 0/0\ntmp = 16'F(0xfe00)\nelsif ((64'F(abs(S1.f16)) == +INF) && (64'F(abs(S2.f16)) == +INF)) then\n// inf/inf\ntmp = 16'F(0xfe00)\nelsif ((64'F(S1.f16) == 0.0) || (64'F(abs(S2.f16)) == +INF)) then\n// x/0, or inf/y\ntmp = sign_out ? -INF : +INF\nelsif ((64'F(abs(S1.f16)) == +INF) || (64'F(S2.f16) == 0.0)) then\n// x/inf, 0/y\ntmp = sign_out ? -0.0 : 0.0\nelse\ntmp = sign_out ? -abs(S0.f16) : abs(S0.f16)\nendif;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
-  VOP3AOp.V_CVT_PKACCUM_U8_F32: "byte = S1.u32[1 : 0];\nbit = byte.u32 * 8U;\nD0.u32[bit + 7U : bit] = 32'U(f32_to_u8(S0.f32))",
-  VOP3AOp.V_MAD_U32_U16: "D0.u32 = 32'U(S0.u16) * 32'U(S1.u16) + S2.u32",
-  VOP3AOp.V_MAD_I32_I16: "D0.i32 = 32'I(S0.i16) * 32'I(S1.i16) + S2.i32",
-  VOP3AOp.V_XAD_U32: 'D0.u32 = (S0.u32 ^ S1.u32) + S2.u32',
-  VOP3AOp.V_MIN3_F16: 'D0.f16 = v_min_f16(v_min_f16(S0.f16, S1.f16), S2.f16)',
-  VOP3AOp.V_MIN3_I16: 'D0.i16 = v_min_i16(v_min_i16(S0.i16, S1.i16), S2.i16)',
-  VOP3AOp.V_MIN3_U16: 'D0.u16 = v_min_u16(v_min_u16(S0.u16, S1.u16), S2.u16)',
-  VOP3AOp.V_MAX3_F16: 'D0.f16 = v_max_f16(v_max_f16(S0.f16, S1.f16), S2.f16)',
-  VOP3AOp.V_MAX3_I16: 'D0.i16 = v_max_i16(v_max_i16(S0.i16, S1.i16), S2.i16)',
-  VOP3AOp.V_MAX3_U16: 'D0.u16 = v_max_u16(v_max_u16(S0.u16, S1.u16), S2.u16)',
-  VOP3AOp.V_MED3_F16: "if (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)) || isNAN(64'F(S2.f16))) then\nD0.f16 = v_min3_f16(S0.f16, S1.f16, S2.f16)\nelsif v_max3_f16(S0.f16, S1.f16, S2.f16) == S0.f16 then\nD0.f16 = v_max_f16(S1.f16, S2.f16)\nelsif v_max3_f16(S0.f16, S1.f16, S2.f16) == S1.f16 then\nD0.f16 = v_max_f16(S0.f16, S2.f16)\nelse\nD0.f16 = v_max_f16(S0.f16, S1.f16)\nendif",
-  VOP3AOp.V_MED3_I16: 'if v_max3_i16(S0.i16, S1.i16, S2.i16) == S0.i16 then\nD0.i16 = v_max_i16(S1.i16, S2.i16)\nelsif v_max3_i16(S0.i16, S1.i16, S2.i16) == S1.i16 then\nD0.i16 = v_max_i16(S0.i16, S2.i16)\nelse\nD0.i16 = v_max_i16(S0.i16, S1.i16)\nendif',
-  VOP3AOp.V_MED3_U16: 'if v_max3_u16(S0.u16, S1.u16, S2.u16) == S0.u16 then\nD0.u16 = v_max_u16(S1.u16, S2.u16)\nelsif v_max3_u16(S0.u16, S1.u16, S2.u16) == S1.u16 then\nD0.u16 = v_max_u16(S0.u16, S2.u16)\nelse\nD0.u16 = v_max_u16(S0.u16, S1.u16)\nendif',
-  VOP3AOp.V_LSHL_ADD_U32: 'D0.u32 = (S0.u32 << S1.u32[4 : 0].u32) + S2.u32',
-  VOP3AOp.V_ADD_LSHL_U32: 'D0.u32 = ((S0.u32 + S1.u32) << S2.u32[4 : 0].u32)',
-  VOP3AOp.V_ADD3_U32: 'D0.u32 = S0.u32 + S1.u32 + S2.u32',
-  VOP3AOp.V_LSHL_OR_B32: 'D0.u32 = ((S0.u32 << S1.u32[4 : 0].u32) | S2.u32)',
-  VOP3AOp.V_AND_OR_B32: 'D0.u32 = ((S0.u32 & S1.u32) | S2.u32)',
-  VOP3AOp.V_OR3_B32: 'D0.u32 = (S0.u32 | S1.u32 | S2.u32)',
-  VOP3AOp.V_MAD_F16: 'D0.f16 = S0.f16 * S1.f16 + S2.f16',
-  VOP3AOp.V_MAD_U16: 'D0.u16 = S0.u16 * S1.u16 + S2.u16',
-  VOP3AOp.V_MAD_I16: 'D0.i16 = S0.i16 * S1.i16 + S2.i16',
-  VOP3AOp.V_FMA_F16: 'D0.f16 = fma(S0.f16, S1.f16, S2.f16)',
-  VOP3AOp.V_DIV_FIXUP_F16: "sign_out = (sign(S1.f16) ^ sign(S2.f16));\nif isNAN(64'F(S2.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S2.f16)))\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif ((64'F(S1.f16) == 0.0) && (64'F(S2.f16) == 0.0)) then\n// 0/0\nD0.f16 = 16'F(0xfe00)\nelsif ((64'F(abs(S1.f16)) == +INF) && (64'F(abs(S2.f16)) == +INF)) then\n// inf/inf\nD0.f16 = 16'F(0xfe00)\nelsif ((64'F(S1.f16) == 0.0) || (64'F(abs(S2.f16)) == +INF)) then\n// x/0, or inf/y\nD0.f16 = sign_out ? -INF.f16 : +INF.f16\nelsif ((64'F(abs(S1.f16)) == +INF) || (64'F(S2.f16) == 0.0)) then\n// x/inf, 0/y\nD0.f16 = sign_out ? -16'0.0 : 16'0.0\nelse\nD0.f16 = sign_out ? -abs(S0.f16) : abs(S0.f16)\nendif",
-  VOP3AOp.V_LSHL_ADD_U64: 'D0.u64 = (S0.u64 << S1.u32[2 : 0].u32) + S2.u64',
-  VOP3AOp.V_BITOP3_B16: "TTBL = { INST.OMOD[1 : 0], INST.ABS[2 : 0], INST.NEG[2 : 0] };\ntmp = 16'0U;\ntmp = (tmp | (32'I(TTBL.b32 & 0x1) != 0 ? 16'U(~S0.b16 & ~S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x2) != 0 ? 16'U(~S0.b16 & ~S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x4) != 0 ? 16'U(~S0.b16 & S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x8) != 0 ? 16'U(~S0.b16 & S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x10) != 0 ? 16'U(S0.b16 & ~S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x20) != 0 ? 16'U(S0.b16 & ~S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x40) != 0 ? 16'U(S0.b16 & S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x80) != 0 ? 16'U(S0.b16 & S1.b16 & S2.b16) : 16'0U));\nD.b16 = tmp.b16",
-  VOP3AOp.V_BITOP3_B32: "TTBL = { INST.OMOD[1 : 0], INST.ABS[2 : 0], INST.NEG[2 : 0] };\ntmp = 0U;\ntmp = (tmp | (32'I(TTBL.b32 & 0x1) != 0 ? 32'U(~S0.b32 & ~S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x2) != 0 ? 32'U(~S0.b32 & ~S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x4) != 0 ? 32'U(~S0.b32 & S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x8) != 0 ? 32'U(~S0.b32 & S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x10) != 0 ? 32'U(S0.b32 & ~S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x20) != 0 ? 32'U(S0.b32 & ~S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x40) != 0 ? 32'U(S0.b32 & S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x80) != 0 ? 32'U(S0.b32 & S1.b32 & S2.b32) : 0U));\nD.b32 = tmp.b32",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP8_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_fp8_scale(S0.f32, scale.u8);\ntmp1 = f32_to_fp8_scale(S1.f32, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_BF8_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_bf8_scale(S0.f32, scale.u8);\ntmp1 = f32_to_bf8_scale(S1.f32, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_FP8_F32: "scale = 32'U(exponent(S2.f32));\ntmp = f32_to_fp8_sr_scale(S0.f32, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_BF8_F32: "scale = 32'U(exponent(S2.f32));\ntmp = f32_to_bf8_sr_scale(S0.f32, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_F32_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_f32_scale(src[7 : 0].fp8, scale.u8);\ntmp1 = fp8_to_f32_scale(src[15 : 8].fp8, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_PK_F32_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_f32_scale(src[7 : 0].bf8, scale.u8);\ntmp1 = bf8_to_f32_scale(src[15 : 8].bf8, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_F32_FP8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].fp8;\ntmp = fp8_to_f32_scale(src, scale.u8);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_SCALEF32_F32_BF8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].bf8;\ntmp = bf8_to_f32_scale(src, scale.u8);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP4_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_fp4_scale(S0.f32, scale.u8);\ntmp1 = f32_to_fp4_scale(S1.f32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK_FP4_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = f32_to_fp4_sr_scale(S0[31 : 0].f32, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = f32_to_fp4_sr_scale(S0[63 : 32].f32, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_F32_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_f32_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_f32_scale(src[7 : 4].fp4, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP8_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_fp8_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_fp8_scale(S0[31 : 16].f16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_BF8_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_bf8_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_bf8_scale(S0[31 : 16].f16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_FP8_F16: "scale = 32'U(exponent(S2.f32));\ntmp = f16_to_fp8_sr_scale(S0.f16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_BF8_F16: "scale = 32'U(exponent(S2.f32));\ntmp = f16_to_bf8_sr_scale(S0.f16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP8_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_fp8_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_fp8_scale(S0[31 : 16].bf16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_BF8_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_bf8_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_bf8_scale(S0[31 : 16].bf16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_FP8_BF16: "scale = 32'U(exponent(S2.f32));\ntmp = bf16_to_fp8_sr_scale(S0.bf16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_BF8_BF16: "scale = 32'U(exponent(S2.f32));\ntmp = bf16_to_bf8_sr_scale(S0.bf16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_F16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_f16_scale(src[7 : 0].fp8, scale.u8);\ntmp1 = fp8_to_f16_scale(src[15 : 8].fp8, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_PK_F16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_f16_scale(src[7 : 0].bf8, scale.u8);\ntmp1 = bf8_to_f16_scale(src[15 : 8].bf8, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_F16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].fp8;\ntmp = fp8_to_f16_scale(src, scale.u8);\n// OPSEL[3] controls destination hi/lo\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_SCALEF32_F16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].bf8;\ntmp = bf8_to_f16_scale(src, scale.u8);\n// OPSEL[3] controls destination hi/lo\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP4_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_fp4_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_fp4_scale(S0[31 : 16].f16, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_FP4_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_fp4_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_fp4_scale(S0[31 : 16].bf16, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK_FP4_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = f16_to_fp4_sr_scale(S0[15 : 0].f16, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = f16_to_fp4_sr_scale(S0[31 : 16].f16, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK_FP4_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = bf16_to_fp4_sr_scale(S0[15 : 0].bf16, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = bf16_to_fp4_sr_scale(S0[31 : 16].bf16, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
-  VOP3AOp.V_CVT_SCALEF32_PK_F16_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_f16_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_f16_scale(src[7 : 4].fp4, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_PK_BF16_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_bf16_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_bf16_scale(src[7 : 4].fp4, scale.u8);\nD0[15 : 0].bf16 = tmp0;\nD0[31 : 16].bf16 = tmp1",
-  VOP3AOp.V_CVT_SCALEF32_2XPK16_FP6_F32: "scale = 32'U(exponent(S2.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 15 do\ndOffset = pass * 12;\nsOffset = pass * 32;\n// Note that S0 and S1 inputs are interleaved in the packed result.\ntmp[dOffset + 5 : dOffset].fp6 = f32_to_fp6_scale(S0[sOffset + 31 : sOffset].f32, scale.u8);\ntmp[dOffset + 11 : dOffset + 6].fp6 = f32_to_fp6_scale(S1[sOffset + 31 : sOffset].f32, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_2XPK16_BF6_F32: "scale = 32'U(exponent(S2.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 15 do\ndOffset = pass * 12;\nsOffset = pass * 32;\n// Note that S0 and S1 inputs are interleaved in the packed result.\ntmp[dOffset + 5 : dOffset].bf6 = f32_to_bf6_scale(S0[sOffset + 31 : sOffset].f32, scale.u8);\ntmp[dOffset + 11 : dOffset + 6].bf6 = f32_to_bf6_scale(S1[sOffset + 31 : sOffset].f32, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_FP6_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 32;\ntmp[dOffset + 5 : dOffset].fp6 = f32_to_fp6_sr_scale(S0[sOffset + 31 : sOffset].f32, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_BF6_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 32;\ntmp[dOffset + 5 : dOffset].bf6 = f32_to_bf6_sr_scale(S0[sOffset + 31 : sOffset].f32, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_PK32_F32_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 1024'B;\nfor pass in 0 : 31 do\ndOffset = pass * 32;\nsOffset = pass * 6;\ntmp[dOffset + 31 : dOffset].f32 = fp6_to_f32_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[1023 : 0] = tmp.b1024",
-  VOP3AOp.V_CVT_SCALEF32_PK32_F32_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 1024'B;\nfor pass in 0 : 31 do\ndOffset = pass * 32;\nsOffset = pass * 6;\ntmp[dOffset + 31 : dOffset].f32 = bf6_to_f32_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[1023 : 0] = tmp.b1024",
-  VOP3AOp.V_CVT_SCALEF32_PK32_FP6_F16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = f16_to_fp6_scale(S0[sOffset + 15 : sOffset].f16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_PK32_FP6_BF16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = bf16_to_fp6_scale(S0[sOffset + 15 : sOffset].bf16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_PK32_BF6_F16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = f16_to_bf6_scale(S0[sOffset + 15 : sOffset].f16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_PK32_BF6_BF16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = bf16_to_bf6_scale(S0[sOffset + 15 : sOffset].bf16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_FP6_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = f16_to_fp6_sr_scale(S0[sOffset + 15 : sOffset].f16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_FP6_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = bf16_to_fp6_sr_scale(S0[sOffset + 15 : sOffset].bf16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_BF6_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = f16_to_bf6_sr_scale(S0[sOffset + 15 : sOffset].f16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_SR_PK32_BF6_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = bf16_to_bf6_sr_scale(S0[sOffset + 15 : sOffset].bf16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
-  VOP3AOp.V_CVT_SCALEF32_PK32_F16_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].f16 = fp6_to_f16_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
-  VOP3AOp.V_CVT_SCALEF32_PK32_BF16_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].bf16 = fp6_to_bf16_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
-  VOP3AOp.V_CVT_SCALEF32_PK32_F16_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].f16 = bf6_to_f16_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
-  VOP3AOp.V_CVT_SCALEF32_PK32_BF16_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].bf16 = bf6_to_bf16_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
-  VOP3AOp.V_ASHR_PK_I8_I32: "SAT8 = lambda(n) (\nif n <= -128 then\nreturn 8'0x80\nelsif n >= 127 then\nreturn 8'0x7f\nelse\nreturn n[7 : 0].b8\nendif);\ndeclare tmp : 16'B;\ntmp[7 : 0] = SAT8(S0.i32 >> S2[4 : 0].u32);\ntmp[15 : 8] = SAT8(S1.i32 >> S2[4 : 0].u32);\nD0[15 : 0] = tmp",
-  VOP3AOp.V_ASHR_PK_U8_I32: "SAT8 = lambda(n) (\nif n <= 0 then\nreturn 8'0x0\nelsif n >= 255 then\nreturn 8'0xff\nelse\nreturn n[7 : 0].b8\nendif);\ndeclare tmp : 16'B;\ntmp[7 : 0] = SAT8(S0.i32 >> S2[4 : 0].u32);\ntmp[15 : 8] = SAT8(S1.i32 >> S2[4 : 0].u32);\nD0[15 : 0] = tmp",
-  VOP3AOp.V_CVT_PK_F16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ntmp[15 : 0].f16 = f32_to_f16(S0.f32);\ntmp[31 : 16].f16 = f32_to_f16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode',
-  VOP3AOp.V_CVT_PK_BF16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ntmp[15 : 0].bf16 = f32_to_bf16(S0.f32);\ntmp[31 : 16].bf16 = f32_to_bf16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode',
-  VOP3AOp.V_CVT_SCALEF32_PK_BF16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_bf16_scale(src[7 : 0].fp8, scale);\ntmp1 = fp8_to_bf16_scale(src[15 : 8].fp8, scale);\nD0[15 : 0].bf16 = tmp0.bf16;\nD0[31 : 16].bf16 = tmp1.bf16",
-  VOP3AOp.V_CVT_SCALEF32_PK_BF16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_bf16_scale(src[7 : 0].bf8, scale);\ntmp1 = bf8_to_bf16_scale(src[15 : 8].bf8, scale);\nD0[15 : 0].bf16 = tmp0.bf16;\nD0[31 : 16].bf16 = tmp1.bf16",
-  VOP3AOp.V_ADD_F64: 'D0.f64 = S0.f64 + S1.f64',
-  VOP3AOp.V_MUL_F64: 'D0.f64 = S0.f64 * S1.f64',
-  VOP3AOp.V_MIN_F64: "if (WAVE_MODE.IEEE && isSignalNAN(S0.f64)) then\nD0.f64 = cvtToQuietNAN(S0.f64)\nelsif (WAVE_MODE.IEEE && isSignalNAN(S1.f64)) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif isNAN(S0.f64) then\nD0.f64 = S1.f64\nelsif isNAN(S1.f64) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == +0.0) && (S1.f64 == -0.0)) then\nD0.f64 = S1.f64\nelsif ((S0.f64 == -0.0) && (S1.f64 == +0.0)) then\nD0.f64 = S0.f64\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F64.\nD0.f64 = S0.f64 < S1.f64 ? S0.f64 : S1.f64\nendif",
-  VOP3AOp.V_MAX_F64: 'if (WAVE_MODE.IEEE && isSignalNAN(S0.f64)) then\nD0.f64 = cvtToQuietNAN(S0.f64)\nelsif (WAVE_MODE.IEEE && isSignalNAN(S1.f64)) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif isNAN(S0.f64) then\nD0.f64 = S1.f64\nelsif isNAN(S1.f64) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == +0.0) && (S1.f64 == -0.0)) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == -0.0) && (S1.f64 == +0.0)) then\nD0.f64 = S1.f64\nelsif WAVE_MODE.IEEE then\nD0.f64 = S0.f64 >= S1.f64 ? S0.f64 : S1.f64\nelse\nD0.f64 = S0.f64 > S1.f64 ? S0.f64 : S1.f64\nendif',
-  VOP3AOp.V_LDEXP_F64: 'D0.f64 = S0.f64 * 2.0 ** S1.i32',
-  VOP3AOp.V_MUL_LO_U32: 'D0.u32 = S0.u32 * S1.u32',
-  VOP3AOp.V_MUL_HI_U32: "D0.u32 = 32'U((64'U(S0.u32) * 64'U(S1.u32)) >> 32U)",
-  VOP3AOp.V_MUL_HI_I32: "D0.i32 = 32'I((64'I(S0.i32) * 64'I(S1.i32)) >> 32U)",
-  VOP3AOp.V_LDEXP_F32: 'D0.f32 = S0.f32 * 2.0F ** S1.i32',
-  VOP3AOp.V_READLANE_B32: 'lane = S1.u32[5 : 0];\n// Lane select\nD0.b32 = VGPR[lane][SRC0.u32]',
-  VOP3AOp.V_WRITELANE_B32: 'lane = S1.u32[5 : 0];\n// Lane select\nVGPR[lane][VDST.u32] = S0.b32',
-  VOP3AOp.V_BCNT_U32_B32: "tmp = S1.u32;\nfor i in 0 : 31 do\ntmp += S0[i].u32;\n// count i'th bit\nendfor;\nD0.u32 = tmp",
-  VOP3AOp.V_MBCNT_LO_U32_B32: "ThreadMask = (1LL << laneId.u32) - 1LL;\nMaskedValue = (S0.u32 & ThreadMask[31 : 0].u32);\ntmp = S1.u32;\nfor i in 0 : 31 do\ntmp += MaskedValue[i] == 1'1U ? 1U : 0U\nendfor;\nD0.u32 = tmp",
-  VOP3AOp.V_MBCNT_HI_U32_B32: "ThreadMask = (1LL << laneId.u32) - 1LL;\nMaskedValue = (S0.u32 & ThreadMask[63 : 32].u32);\ntmp = S1.u32;\nfor i in 0 : 31 do\ntmp += MaskedValue[i] == 1'1U ? 1U : 0U\nendfor;\nD0.u32 = tmp",
-  VOP3AOp.V_LSHLREV_B64: 'D0.u64 = (S1.u64 << S0[5 : 0].u32)',
-  VOP3AOp.V_LSHRREV_B64: 'D0.u64 = (S1.u64 >> S0[5 : 0].u32)',
-  VOP3AOp.V_ASHRREV_I64: 'D0.i64 = (S1.i64 >> S0[5 : 0].u32)',
-  VOP3AOp.V_TRIG_PREOP_F64: "shift = 32'I(S1[4 : 0].u32) * 53;\nif exponent(S0.f64) > 1077 then\nshift += exponent(S0.f64) - 1077\nendif;\n// (2.0/PI) == 0.{b_1200, b_1199, b_1198, ..., b_1, b_0}\n// b_1200 is the MSB of the fractional part of 2.0/PI\n// Left shift operation indicates which bits are brought\n// into the whole part of the number.\n// Only whole part of result is kept.\nresult = 64'F((1201'B(2.0 / PI)[1200 : 0] << shift.u32) & 1201'0x1fffffffffffff);\nscale = -53 - shift;\nif exponent(S0.f64) >= 1968 then\nscale += 128\nendif;\nD0.f64 = ldexp(result, scale)",
-  VOP3AOp.V_BFM_B32: 'D0.u32 = (((1U << S0[4 : 0].u32) - 1U) << S1[4 : 0].u32)',
-  VOP3AOp.V_CVT_PKNORM_I16_F32: "declare tmp : 32'B;\ntmp[15 : 0].i16 = f32_to_snorm(S0.f32);\ntmp[31 : 16].i16 = f32_to_snorm(S1.f32);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_PKNORM_U16_F32: "declare tmp : 32'B;\ntmp[15 : 0].u16 = f32_to_unorm(S0.f32);\ntmp[31 : 16].u16 = f32_to_unorm(S1.f32);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_PKRTZ_F16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_TOWARD_ZERO;\ntmp[15 : 0].f16 = f32_to_f16(S0.f32);\ntmp[31 : 16].f16 = f32_to_f16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode;\n// Round-toward-zero regardless of current round mode setting in hardware.',
-  VOP3AOp.V_CVT_PK_U16_U32: "declare tmp : 32'B;\ntmp[15 : 0].u16 = u32_to_u16(S0.u32);\ntmp[31 : 16].u16 = u32_to_u16(S1.u32);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_PK_I16_I32: "declare tmp : 32'B;\ntmp[15 : 0].i16 = i32_to_i16(S0.i32);\ntmp[31 : 16].i16 = i32_to_i16(S1.i32);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_PKNORM_I16_F16: "declare tmp : 32'B;\ntmp[15 : 0].i16 = f16_to_snorm(S0.f16);\ntmp[31 : 16].i16 = f16_to_snorm(S1.f16);\nD0 = tmp.b32",
-  VOP3AOp.V_CVT_PKNORM_U16_F16: "declare tmp : 32'B;\ntmp[15 : 0].u16 = f16_to_unorm(S0.f16);\ntmp[31 : 16].u16 = f16_to_unorm(S1.f16);\nD0 = tmp.b32",
-  VOP3AOp.V_ADD_I32: 'D0.i32 = S0.i32 + S1.i32',
-  VOP3AOp.V_SUB_I32: 'D0.i32 = S0.i32 - S1.i32',
-  VOP3AOp.V_ADD_I16: 'D0.i16 = S0.i16 + S1.i16',
-  VOP3AOp.V_SUB_I16: 'D0.i16 = S0.i16 - S1.i16',
-  VOP3AOp.V_PACK_B32_F16: 'D0[31 : 16].f16 = S1.f16;\nD0[15 : 0].f16 = S0.f16',
-  VOP3AOp.V_MUL_LEGACY_F32: "if ((64'F(S0.f32) == 0.0) || (64'F(S1.f32) == 0.0)) then\n// DX9 rules, 0.0 * x = 0.0\nD0.f32 = 0.0F\nelse\nD0.f32 = S0.f32 * S1.f32\nendif",
-  VOP3AOp.V_CVT_PK_FP8_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u32 == 0U then\nVGPR[laneId][VDST.u32][15 : 0].b16 = { f32_to_fp8(S1.f32), f32_to_fp8(S0.f32) };\n// D0[31:16] are preserved\nelse\nVGPR[laneId][VDST.u32][31 : 16].b16 = { f32_to_fp8(S1.f32), f32_to_fp8(S0.f32) };\n// D0[15:0] are preserved\nendif;\nROUND_MODE = prev_mode',
-  VOP3AOp.V_CVT_PK_BF8_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u32 == 0U then\nVGPR[laneId][VDST.u32][15 : 0].b16 = { f32_to_bf8(S1.f32), f32_to_bf8(S0.f32) };\n// D0[31:16] are preserved\nelse\nVGPR[laneId][VDST.u32][31 : 16].b16 = { f32_to_bf8(S1.f32), f32_to_bf8(S0.f32) };\n// D0[15:0] are preserved\nendif;\nROUND_MODE = prev_mode',
-  VOP3AOp.V_CVT_SR_FP8_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ns = sign(S0.f32);\ne = exponent(S0.f32);\nm = 23'U(32'U(23'B(mantissa(S0.f32))) + S1[31 : 12].u32);\ntmp = float32(s, e, m);\n// Add stochastic value to mantissa, wrap around on overflow\nif OPSEL[3 : 2].u2 == 2'0U then\nVGPR[laneId][VDST.u32][7 : 0].fp8 = f32_to_fp8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'1U then\nVGPR[laneId][VDST.u32][15 : 8].fp8 = f32_to_fp8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'2U then\nVGPR[laneId][VDST.u32][23 : 16].fp8 = f32_to_fp8(tmp.f32)\nelse\nVGPR[laneId][VDST.u32][31 : 24].fp8 = f32_to_fp8(tmp.f32)\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
-  VOP3AOp.V_CVT_SR_BF8_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ns = sign(S0.f32);\ne = exponent(S0.f32);\nm = 23'U(32'U(23'B(mantissa(S0.f32))) + S1[31 : 11].u32);\ntmp = float32(s, e, m);\n// Add stochastic value to mantissa, wrap around on overflow\nif OPSEL[3 : 2].u2 == 2'0U then\nVGPR[laneId][VDST.u32][7 : 0].bf8 = f32_to_bf8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'1U then\nVGPR[laneId][VDST.u32][15 : 8].bf8 = f32_to_bf8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'2U then\nVGPR[laneId][VDST.u32][23 : 16].bf8 = f32_to_bf8(tmp.f32)\nelse\nVGPR[laneId][VDST.u32][31 : 24].bf8 = f32_to_bf8(tmp.f32)\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
-  VOP3AOp.V_CVT_SR_F16_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u2 == 2'0U then\nVGPR[laneId][VDST.u32][15 : 0].f16 = 16'F(f32_to_f16_SR(S0.f32, S1.u32))\nelse\nVGPR[laneId][VDST.u32][31 : 16].f16 = 16'F(f32_to_f16_sr(S0.f32, S1.u32))\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
-  VOP3AOp.V_CVT_SR_BF16_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u2 == 2'0U then\nVGPR[laneId][VDST.u32][15 : 0].bf16 = 16'BF(f32_to_bf16_SR(S0.f32, S1.u32))\nelse\nVGPR[laneId][VDST.u32][31 : 16].bf16 = 16'BF(f32_to_bf16_sr(S0.f32, S1.u32))\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
-  VOP3AOp.V_MINIMUM3_F32: "D0.f32 = 32'F(v_minimum_f32(v_minimum_f32(S0.f32, S1.f32), S2.f32))",
-  VOP3AOp.V_MAXIMUM3_F32: "D0.f32 = 32'F(v_maximum_f32(v_maximum_f32(S0.f32, S1.f32), S2.f32))",
-  VOP3BOp.V_ADD_CO_U32: "tmp = 64'U(S0.u32) + 64'U(S1.u32);\nVCC.u64[laneId] = tmp >= 0x100000000ULL ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_ADDC_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_SUB_CO_U32: "tmp = S0.u32 - S1.u32;\nVCC.u64[laneId] = S1.u32 > S0.u32 ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_SUBREV_CO_U32: "tmp = S1.u32 - S0.u32;\nVCC.u64[laneId] = S0.u32 > S1.u32 ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_ADDC_CO_U32: "tmp = 64'U(S0.u32) + 64'U(S1.u32) + VCC.u64[laneId].u64;\nVCC.u64[laneId] = tmp >= 0x100000000ULL ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_ADDC_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_SUBB_CO_U32: "tmp = S0.u32 - S1.u32 - VCC.u64[laneId].u32;\nVCC.u64[laneId] = 64'U(S1.u32) + VCC.u64[laneId].u64 > 64'U(S0.u32) ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_SUBBREV_CO_U32: "tmp = S1.u32 - S0.u32 - VCC.u64[laneId].u32;\nVCC.u64[laneId] = 64'U(S0.u32) + VCC.u64[laneId].u64 > 64'U(S1.u32) ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
-  VOP3BOp.V_DIV_SCALE_F32: "VCC = 0x0LL;\nif ((64'F(S2.f32) == 0.0) || (64'F(S1.f32) == 0.0)) then\nD0.f32 = NAN.f32\nelsif exponent(S2.f32) - exponent(S1.f32) >= 96 then\n// N/D near MAX_FLOAT_F32\nVCC = 0x1LL;\nif S0.f32 == S1.f32 then\n// Only scale the denominator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif S1.f32 == DENORM.f32 then\nD0.f32 = ldexp(S0.f32, 64)\nelsif ((1.0 / 64'F(S1.f32) == DENORM.f64) && (S2.f32 / S1.f32 == DENORM.f32)) then\nVCC = 0x1LL;\nif S0.f32 == S1.f32 then\n// Only scale the denominator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif 1.0 / 64'F(S1.f32) == DENORM.f64 then\nD0.f32 = ldexp(S0.f32, -64)\nelsif S2.f32 / S1.f32 == DENORM.f32 then\nVCC = 0x1LL;\nif S0.f32 == S2.f32 then\n// Only scale the numerator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif exponent(S2.f32) <= 23 then\n// Numerator is tiny\nD0.f32 = ldexp(S0.f32, 64)\nendif",
-  VOP3BOp.V_DIV_SCALE_F64: 'VCC = 0x0LL;\nif ((S2.f64 == 0.0) || (S1.f64 == 0.0)) then\nD0.f64 = NAN.f64\nelsif exponent(S2.f64) - exponent(S1.f64) >= 768 then\n// N/D near MAX_FLOAT_F64\nVCC = 0x1LL;\nif S0.f64 == S1.f64 then\n// Only scale the denominator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif S1.f64 == DENORM.f64 then\nD0.f64 = ldexp(S0.f64, 128)\nelsif ((1.0 / S1.f64 == DENORM.f64) && (S2.f64 / S1.f64 == DENORM.f64)) then\nVCC = 0x1LL;\nif S0.f64 == S1.f64 then\n// Only scale the denominator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif 1.0 / S1.f64 == DENORM.f64 then\nD0.f64 = ldexp(S0.f64, -128)\nelsif S2.f64 / S1.f64 == DENORM.f64 then\nVCC = 0x1LL;\nif S0.f64 == S2.f64 then\n// Only scale the numerator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif exponent(S2.f64) <= 53 then\n// Numerator is tiny\nD0.f64 = ldexp(S0.f64, 128)\nendif',
-  VOP3BOp.V_MAD_U64_U32: "{ D1.u1, D0.u64 } = 65'B(65'U(S0.u32) * 65'U(S1.u32) + 65'U(S2.u64))",
-  VOP3BOp.V_MAD_I64_I32: "{ D1.i1, D0.i64 } = 65'B(65'I(S0.i32) * 65'I(S1.i32) + 65'I(S2.i64))",
+  VOP3Op.V_CMP_CLASS_F32_E64: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_CLASS_F32_E64: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
+  VOP3Op.V_CMP_CLASS_F64_E64: "declare result : 1'U;\nif isSignalNAN(S0.f64) then\nresult = S1.u32[0]\nelsif isQuietNAN(S0.f64) then\nresult = S1.u32[1]\nelsif exponent(S0.f64) == 2047 then\n// +-INF\nresult = S1.u32[sign(S0.f64) ? 2 : 9]\nelsif exponent(S0.f64) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f64) ? 3 : 8]\nelsif abs(S0.f64) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f64) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f64) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_CLASS_F64_E64: "declare result : 1'U;\nif isSignalNAN(S0.f64) then\nresult = S1.u32[0]\nelsif isQuietNAN(S0.f64) then\nresult = S1.u32[1]\nelsif exponent(S0.f64) == 2047 then\n// +-INF\nresult = S1.u32[sign(S0.f64) ? 2 : 9]\nelsif exponent(S0.f64) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f64) ? 3 : 8]\nelsif abs(S0.f64) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f64) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f64) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
+  VOP3Op.V_CMP_CLASS_F16_E64: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f16)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f16)) then\nresult = S1.u32[1]\nelsif exponent(S0.f16) == 31 then\n// +-INF\nresult = S1.u32[sign(S0.f16) ? 2 : 9]\nelsif exponent(S0.f16) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f16) ? 3 : 8]\nelsif 64'F(abs(S0.f16)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f16) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f16) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_CLASS_F16_E64: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f16)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f16)) then\nresult = S1.u32[1]\nelsif exponent(S0.f16) == 31 then\n// +-INF\nresult = S1.u32[sign(S0.f16) ? 2 : 9]\nelsif exponent(S0.f16) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f16) ? 3 : 8]\nelsif 64'F(abs(S0.f16)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f16) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f16) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
+  VOP3Op.V_CMP_F_F16_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_F16_E64: 'D0.u64[laneId] = S0.f16 < S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_F16_E64: 'D0.u64[laneId] = S0.f16 == S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_F16_E64: 'D0.u64[laneId] = S0.f16 <= S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_F16_E64: 'D0.u64[laneId] = S0.f16 > S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LG_F16_E64: 'D0.u64[laneId] = S0.f16 <> S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_F16_E64: 'D0.u64[laneId] = S0.f16 >= S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_O_F16_E64: "D0.u64[laneId] = (!isNAN(64'F(S0.f16)) && !isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_U_F16_E64: "D0.u64[laneId] = (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_NGE_F16_E64: 'D0.u64[laneId] = !(S0.f16 >= S1.f16);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLG_F16_E64: 'D0.u64[laneId] = !(S0.f16 <> S1.f16);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NGT_F16_E64: 'D0.u64[laneId] = !(S0.f16 > S1.f16);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLE_F16_E64: 'D0.u64[laneId] = !(S0.f16 <= S1.f16);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NEQ_F16_E64: 'D0.u64[laneId] = !(S0.f16 == S1.f16);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLT_F16_E64: 'D0.u64[laneId] = !(S0.f16 < S1.f16);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_TRU_F16_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_F16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 < S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 == S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 <= S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 > S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LG_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 <> S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f16 >= S1.f16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_O_F16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(64'F(S0.f16)) && !isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_U_F16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_NGE_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 >= S1.f16);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLG_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 <> S1.f16);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NGT_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 > S1.f16);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLE_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 <= S1.f16);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NEQ_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 == S1.f16);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLT_F16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f16 < S1.f16);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_TRU_F16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_F32_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_F32_E64: 'D0.u64[laneId] = S0.f32 < S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_F32_E64: 'D0.u64[laneId] = S0.f32 == S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_F32_E64: 'D0.u64[laneId] = S0.f32 <= S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_F32_E64: 'D0.u64[laneId] = S0.f32 > S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LG_F32_E64: 'D0.u64[laneId] = S0.f32 <> S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_F32_E64: 'D0.u64[laneId] = S0.f32 >= S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_O_F32_E64: "D0.u64[laneId] = (!isNAN(64'F(S0.f32)) && !isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_U_F32_E64: "D0.u64[laneId] = (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_NGE_F32_E64: 'D0.u64[laneId] = !(S0.f32 >= S1.f32);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLG_F32_E64: 'D0.u64[laneId] = !(S0.f32 <> S1.f32);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NGT_F32_E64: 'D0.u64[laneId] = !(S0.f32 > S1.f32);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLE_F32_E64: 'D0.u64[laneId] = !(S0.f32 <= S1.f32);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NEQ_F32_E64: 'D0.u64[laneId] = !(S0.f32 == S1.f32);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLT_F32_E64: 'D0.u64[laneId] = !(S0.f32 < S1.f32);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_TRU_F32_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_F32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 < S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 == S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 <= S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 > S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LG_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 <> S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f32 >= S1.f32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_O_F32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(64'F(S0.f32)) && !isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_U_F32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)));\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_NGE_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 >= S1.f32);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLG_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 <> S1.f32);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NGT_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 > S1.f32);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLE_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 <= S1.f32);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NEQ_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 == S1.f32);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLT_F32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f32 < S1.f32);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_TRU_F32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_F64_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_F64_E64: 'D0.u64[laneId] = S0.f64 < S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_F64_E64: 'D0.u64[laneId] = S0.f64 == S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_F64_E64: 'D0.u64[laneId] = S0.f64 <= S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_F64_E64: 'D0.u64[laneId] = S0.f64 > S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LG_F64_E64: 'D0.u64[laneId] = S0.f64 <> S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_F64_E64: 'D0.u64[laneId] = S0.f64 >= S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_O_F64_E64: 'D0.u64[laneId] = (!isNAN(S0.f64) && !isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_U_F64_E64: 'D0.u64[laneId] = (isNAN(S0.f64) || isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NGE_F64_E64: 'D0.u64[laneId] = !(S0.f64 >= S1.f64);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLG_F64_E64: 'D0.u64[laneId] = !(S0.f64 <> S1.f64);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NGT_F64_E64: 'D0.u64[laneId] = !(S0.f64 > S1.f64);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLE_F64_E64: 'D0.u64[laneId] = !(S0.f64 <= S1.f64);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NEQ_F64_E64: 'D0.u64[laneId] = !(S0.f64 == S1.f64);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NLT_F64_E64: 'D0.u64[laneId] = !(S0.f64 < S1.f64);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_TRU_F64_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_F64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 < S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 == S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 <= S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 > S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LG_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 <> S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.f64 >= S1.f64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_O_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = (!isNAN(S0.f64) && !isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_U_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = (isNAN(S0.f64) || isNAN(S1.f64));\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NGE_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 >= S1.f64);\n// With NAN inputs this is not the same operation as <\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLG_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 <> S1.f64);\n// With NAN inputs this is not the same operation as ==\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NGT_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 > S1.f64);\n// With NAN inputs this is not the same operation as <=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLE_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 <= S1.f64);\n// With NAN inputs this is not the same operation as >\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NEQ_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 == S1.f64);\n// With NAN inputs this is not the same operation as !=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NLT_F64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = !(S0.f64 < S1.f64);\n// With NAN inputs this is not the same operation as >=\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_TRU_F64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_I16_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_I16_E64: 'D0.u64[laneId] = S0.i16 < S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_I16_E64: 'D0.u64[laneId] = S0.i16 == S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_I16_E64: 'D0.u64[laneId] = S0.i16 <= S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_I16_E64: 'D0.u64[laneId] = S0.i16 > S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_I16_E64: 'D0.u64[laneId] = S0.i16 <> S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_I16_E64: 'D0.u64[laneId] = S0.i16 >= S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_I16_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_U16_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_U16_E64: 'D0.u64[laneId] = S0.u16 < S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_U16_E64: 'D0.u64[laneId] = S0.u16 == S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_U16_E64: 'D0.u64[laneId] = S0.u16 <= S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_U16_E64: 'D0.u64[laneId] = S0.u16 > S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_U16_E64: 'D0.u64[laneId] = S0.u16 <> S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_U16_E64: 'D0.u64[laneId] = S0.u16 >= S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_U16_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_I16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 < S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 == S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 <= S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 > S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 <> S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_I16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i16 >= S1.i16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_I16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_U16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 < S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 == S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 <= S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 > S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 <> S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_U16_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u16 >= S1.u16;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_U16_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_I32_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_I32_E64: 'D0.u64[laneId] = S0.i32 < S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_I32_E64: 'D0.u64[laneId] = S0.i32 == S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_I32_E64: 'D0.u64[laneId] = S0.i32 <= S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_I32_E64: 'D0.u64[laneId] = S0.i32 > S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_I32_E64: 'D0.u64[laneId] = S0.i32 <> S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_I32_E64: 'D0.u64[laneId] = S0.i32 >= S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_I32_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_U32_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_U32_E64: 'D0.u64[laneId] = S0.u32 < S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_U32_E64: 'D0.u64[laneId] = S0.u32 == S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_U32_E64: 'D0.u64[laneId] = S0.u32 <= S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_U32_E64: 'D0.u64[laneId] = S0.u32 > S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_U32_E64: 'D0.u64[laneId] = S0.u32 <> S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_U32_E64: 'D0.u64[laneId] = S0.u32 >= S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_U32_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_I32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 < S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 == S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 <= S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 > S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 <> S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_I32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i32 >= S1.i32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_I32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_U32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 < S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 == S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 <= S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 > S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 <> S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_U32_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u32 >= S1.u32;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_U32_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_I64_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_I64_E64: 'D0.u64[laneId] = S0.i64 < S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_I64_E64: 'D0.u64[laneId] = S0.i64 == S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_I64_E64: 'D0.u64[laneId] = S0.i64 <= S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_I64_E64: 'D0.u64[laneId] = S0.i64 > S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_I64_E64: 'D0.u64[laneId] = S0.i64 <> S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_I64_E64: 'D0.u64[laneId] = S0.i64 >= S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_I64_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_F_U64_E64: "D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMP_LT_U64_E64: 'D0.u64[laneId] = S0.u64 < S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_EQ_U64_E64: 'D0.u64[laneId] = S0.u64 == S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_LE_U64_E64: 'D0.u64[laneId] = S0.u64 <= S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GT_U64_E64: 'D0.u64[laneId] = S0.u64 > S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_NE_U64_E64: 'D0.u64[laneId] = S0.u64 <> S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_GE_U64_E64: 'D0.u64[laneId] = S0.u64 >= S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMP_T_U64_E64: "D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_I64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 < S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 == S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 <= S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 > S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 <> S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_I64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.i64 >= S1.i64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_I64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_F_U64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'0U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CMPX_LT_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 < S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_EQ_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 == S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_LE_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 <= S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GT_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 > S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_NE_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 <> S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_GE_U64_E64: 'EXEC.u64[laneId] = D0.u64[laneId] = S0.u64 >= S1.u64;\n// D0 = VCC in VOPC encoding.',
+  VOP3Op.V_CMPX_T_U64_E64: "EXEC.u64[laneId] = D0.u64[laneId] = 1'1U;\n// D0 = VCC in VOPC encoding.",
+  VOP3Op.V_CNDMASK_B32_E64: 'D0.u32 = VCC.u64[laneId] ? S1.u32 : S0.u32',
+  VOP3Op.V_ADD_F32_E64: 'D0.f32 = S0.f32 + S1.f32',
+  VOP3Op.V_SUB_F32_E64: 'D0.f32 = S0.f32 - S1.f32',
+  VOP3Op.V_SUBREV_F32_E64: 'D0.f32 = S1.f32 - S0.f32',
+  VOP3Op.V_FMAC_F64_E64: 'D0.f64 = fma(S0.f64, S1.f64, D0.f64)',
+  VOP3Op.V_MUL_F32_E64: 'D0.f32 = S0.f32 * S1.f32',
+  VOP3Op.V_MUL_I32_I24_E64: "D0.i32 = 32'I(S0.i24) * 32'I(S1.i24)",
+  VOP3Op.V_MUL_HI_I32_I24_E64: "D0.i32 = 32'I((64'I(S0.i24) * 64'I(S1.i24)) >> 32U)",
+  VOP3Op.V_MUL_U32_U24_E64: "D0.u32 = 32'U(S0.u24) * 32'U(S1.u24)",
+  VOP3Op.V_MUL_HI_U32_U24_E64: "D0.u32 = 32'U((64'U(S0.u24) * 64'U(S1.u24)) >> 32U)",
+  VOP3Op.V_MIN_F32_E64: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S0.f32)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif isNAN(64'F(S0.f32)) then\nD0.f32 = S1.f32\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == +0.0) && (64'F(S1.f32) == -0.0)) then\nD0.f32 = S1.f32\nelsif ((64'F(S0.f32) == -0.0) && (64'F(S1.f32) == +0.0)) then\nD0.f32 = S0.f32\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F32.\nD0.f32 = S0.f32 < S1.f32 ? S0.f32 : S1.f32\nendif",
+  VOP3Op.V_MAX_F32_E64: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S0.f32)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f32))) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif isNAN(64'F(S0.f32)) then\nD0.f32 = S1.f32\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == +0.0) && (64'F(S1.f32) == -0.0)) then\nD0.f32 = S0.f32\nelsif ((64'F(S0.f32) == -0.0) && (64'F(S1.f32) == +0.0)) then\nD0.f32 = S1.f32\nelsif WAVE_MODE.IEEE then\nD0.f32 = S0.f32 >= S1.f32 ? S0.f32 : S1.f32\nelse\nD0.f32 = S0.f32 > S1.f32 ? S0.f32 : S1.f32\nendif",
+  VOP3Op.V_MIN_I32_E64: 'D0.i32 = S0.i32 < S1.i32 ? S0.i32 : S1.i32',
+  VOP3Op.V_MAX_I32_E64: 'D0.i32 = S0.i32 >= S1.i32 ? S0.i32 : S1.i32',
+  VOP3Op.V_MIN_U32_E64: 'D0.u32 = S0.u32 < S1.u32 ? S0.u32 : S1.u32',
+  VOP3Op.V_MAX_U32_E64: 'D0.u32 = S0.u32 >= S1.u32 ? S0.u32 : S1.u32',
+  VOP3Op.V_LSHRREV_B32_E64: 'D0.u32 = (S1.u32 >> S0[4 : 0].u32)',
+  VOP3Op.V_ASHRREV_I32_E64: 'D0.i32 = (S1.i32 >> S0[4 : 0].u32)',
+  VOP3Op.V_LSHLREV_B32_E64: 'D0.u32 = (S1.u32 << S0[4 : 0].u32)',
+  VOP3Op.V_AND_B32_E64: 'D0.u32 = (S0.u32 & S1.u32)',
+  VOP3Op.V_OR_B32_E64: 'D0.u32 = (S0.u32 | S1.u32)',
+  VOP3Op.V_XOR_B32_E64: 'D0.u32 = (S0.u32 ^ S1.u32)',
+  VOP3Op.V_DOT2C_F32_BF16_E64: 'tmp = D0.f32;\ntmp += bf16_to_f32(S0[15 : 0].bf16) * bf16_to_f32(S1[15 : 0].bf16);\ntmp += bf16_to_f32(S0[31 : 16].bf16) * bf16_to_f32(S1[31 : 16].bf16);\nD0.f32 = tmp',
+  VOP3Op.V_ADD_F16_E64: 'D0.f16 = S0.f16 + S1.f16',
+  VOP3Op.V_SUB_F16_E64: 'D0.f16 = S0.f16 - S1.f16',
+  VOP3Op.V_SUBREV_F16_E64: 'D0.f16 = S1.f16 - S0.f16',
+  VOP3Op.V_MUL_F16_E64: 'D0.f16 = S0.f16 * S1.f16',
+  VOP3Op.V_MAC_F16_E64: "tmp = S0.f16 * S1.f16 + D0.f16;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
+  VOP3Op.V_ADD_U16_E64: 'D0.u16 = S0.u16 + S1.u16',
+  VOP3Op.V_SUB_U16_E64: 'D0.u16 = S0.u16 - S1.u16',
+  VOP3Op.V_SUBREV_U16_E64: 'D0.u16 = S1.u16 - S0.u16',
+  VOP3Op.V_MUL_LO_U16_E64: 'D0.u16 = S0.u16 * S1.u16',
+  VOP3Op.V_LSHLREV_B16_E64: 'D0.u16 = (S1.u16 << S0[3 : 0].u32)',
+  VOP3Op.V_LSHRREV_B16_E64: 'D0.u16 = (S1.u16 >> S0[3 : 0].u32)',
+  VOP3Op.V_ASHRREV_I16_E64: 'D0.i16 = (S1.i16 >> S0[3 : 0].u32)',
+  VOP3Op.V_MAX_F16_E64: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == +0.0) && (64'F(S1.f16) == -0.0)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == -0.0) && (64'F(S1.f16) == +0.0)) then\nD0.f16 = S1.f16\nelsif WAVE_MODE.IEEE then\nD0.f16 = S0.f16 >= S1.f16 ? S0.f16 : S1.f16\nelse\nD0.f16 = S0.f16 > S1.f16 ? S0.f16 : S1.f16\nendif",
+  VOP3Op.V_MIN_F16_E64: "if (WAVE_MODE.IEEE && isSignalNAN(64'F(S0.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S0.f16)))\nelsif (WAVE_MODE.IEEE && isSignalNAN(64'F(S1.f16))) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif isNAN(64'F(S0.f16)) then\nD0.f16 = S1.f16\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = S0.f16\nelsif ((64'F(S0.f16) == +0.0) && (64'F(S1.f16) == -0.0)) then\nD0.f16 = S1.f16\nelsif ((64'F(S0.f16) == -0.0) && (64'F(S1.f16) == +0.0)) then\nD0.f16 = S0.f16\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F16.\nD0.f16 = S0.f16 < S1.f16 ? S0.f16 : S1.f16\nendif",
+  VOP3Op.V_MAX_U16_E64: 'D0.u16 = S0.u16 >= S1.u16 ? S0.u16 : S1.u16',
+  VOP3Op.V_MAX_I16_E64: 'D0.i16 = S0.i16 >= S1.i16 ? S0.i16 : S1.i16',
+  VOP3Op.V_MIN_U16_E64: 'D0.u16 = S0.u16 < S1.u16 ? S0.u16 : S1.u16',
+  VOP3Op.V_MIN_I16_E64: 'D0.i16 = S0.i16 < S1.i16 ? S0.i16 : S1.i16',
+  VOP3Op.V_LDEXP_F16_E64: "D0.f16 = S0.f16 * 16'F(2.0F ** 32'I(S1.i16))",
+  VOP3Op.V_ADD_U32_E64: 'D0.u32 = S0.u32 + S1.u32',
+  VOP3Op.V_SUB_U32_E64: 'D0.u32 = S0.u32 - S1.u32',
+  VOP3Op.V_SUBREV_U32_E64: 'D0.u32 = S1.u32 - S0.u32',
+  VOP3Op.V_DOT2C_F32_F16_E64: 'tmp = D0.f32;\ntmp += f16_to_f32(S0[15 : 0].f16) * f16_to_f32(S1[15 : 0].f16);\ntmp += f16_to_f32(S0[31 : 16].f16) * f16_to_f32(S1[31 : 16].f16);\nD0.f32 = tmp',
+  VOP3Op.V_DOT2C_I32_I16_E64: 'tmp = D0.i32;\ntmp += i16_to_i32(S0[15 : 0].i16) * i16_to_i32(S1[15 : 0].i16);\ntmp += i16_to_i32(S0[31 : 16].i16) * i16_to_i32(S1[31 : 16].i16);\nD0.i32 = tmp',
+  VOP3Op.V_DOT4C_I32_I8_E64: 'tmp = D0.i32;\ntmp += i8_to_i32(S0[7 : 0].i8) * i8_to_i32(S1[7 : 0].i8);\ntmp += i8_to_i32(S0[15 : 8].i8) * i8_to_i32(S1[15 : 8].i8);\ntmp += i8_to_i32(S0[23 : 16].i8) * i8_to_i32(S1[23 : 16].i8);\ntmp += i8_to_i32(S0[31 : 24].i8) * i8_to_i32(S1[31 : 24].i8);\nD0.i32 = tmp',
+  VOP3Op.V_DOT8C_I32_I4_E64: 'tmp = D0.i32;\ntmp += i4_to_i32(S0[3 : 0].i4) * i4_to_i32(S1[3 : 0].i4);\ntmp += i4_to_i32(S0[7 : 4].i4) * i4_to_i32(S1[7 : 4].i4);\ntmp += i4_to_i32(S0[11 : 8].i4) * i4_to_i32(S1[11 : 8].i4);\ntmp += i4_to_i32(S0[15 : 12].i4) * i4_to_i32(S1[15 : 12].i4);\ntmp += i4_to_i32(S0[19 : 16].i4) * i4_to_i32(S1[19 : 16].i4);\ntmp += i4_to_i32(S0[23 : 20].i4) * i4_to_i32(S1[23 : 20].i4);\ntmp += i4_to_i32(S0[27 : 24].i4) * i4_to_i32(S1[27 : 24].i4);\ntmp += i4_to_i32(S0[31 : 28].i4) * i4_to_i32(S1[31 : 28].i4);\nD0.i32 = tmp',
+  VOP3Op.V_FMAC_F32_E64: 'D0.f32 = fma(S0.f32, S1.f32, D0.f32)',
+  VOP3Op.V_PK_FMAC_F16_E64: 'D0[15 : 0].f16 = fma(S0[15 : 0].f16, S1[15 : 0].f16, D0[15 : 0].f16);\nD0[31 : 16].f16 = fma(S0[31 : 16].f16, S1[31 : 16].f16, D0[31 : 16].f16)',
+  VOP3Op.V_XNOR_B32_E64: 'D0.u32 = ~(S0.u32 ^ S1.u32)',
+  VOP3Op.V_MAD_I32_I24_E64: "D0.i32 = 32'I(S0.i24) * 32'I(S1.i24) + S2.i32",
+  VOP3Op.V_MAD_U32_U24_E64: "D0.u32 = 32'U(S0.u24) * 32'U(S1.u24) + S2.u32",
+  VOP3Op.V_CUBEID_F32_E64: '// Set D0.f = cubemap face ID ({0.0, 1.0, ..., 5.0}).\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nif S2.f32 < 0.0F then\nD0.f32 = 5.0F\nelse\nD0.f32 = 4.0F\nendif\nelsif abs(S1.f32) >= abs(S0.f32) then\nif S1.f32 < 0.0F then\nD0.f32 = 3.0F\nelse\nD0.f32 = 2.0F\nendif\nelse\nif S0.f32 < 0.0F then\nD0.f32 = 1.0F\nelse\nD0.f32 = 0.0F\nendif\nendif',
+  VOP3Op.V_CUBESC_F32_E64: '// D0.f = cubemap S coordinate.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nif S2.f32 < 0.0F then\nD0.f32 = -S0.f32\nelse\nD0.f32 = S0.f32\nendif\nelsif abs(S1.f32) >= abs(S0.f32) then\nD0.f32 = S0.f32\nelse\nif S0.f32 < 0.0F then\nD0.f32 = S2.f32\nelse\nD0.f32 = -S2.f32\nendif\nendif',
+  VOP3Op.V_CUBETC_F32_E64: '// D0.f = cubemap T coordinate.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nD0.f32 = -S1.f32\nelsif abs(S1.f32) >= abs(S0.f32) then\nif S1.f32 < 0.0F then\nD0.f32 = -S2.f32\nelse\nD0.f32 = S2.f32\nendif\nelse\nD0.f32 = -S1.f32\nendif',
+  VOP3Op.V_CUBEMA_F32_E64: '// D0.f = 2.0 * cubemap major axis.\n// XYZ coordinate is given in (S0.f, S1.f, S2.f).\n// S0.f = x\n// S1.f = y\n// S2.f = z\nif ((abs(S2.f32) >= abs(S0.f32)) && (abs(S2.f32) >= abs(S1.f32))) then\nD0.f32 = S2.f32 * 2.0F\nelsif abs(S1.f32) >= abs(S0.f32) then\nD0.f32 = S1.f32 * 2.0F\nelse\nD0.f32 = S0.f32 * 2.0F\nendif',
+  VOP3Op.V_BFE_U32_E64: 'D0.u32 = ((S0.u32 >> S1[4 : 0].u32) & ((1U << S2[4 : 0].u32) - 1U))',
+  VOP3Op.V_BFE_I32_E64: 'tmp.i32 = ((S0.i32 >> S1[4 : 0].u32) & ((1 << S2[4 : 0].u32) - 1));\nD0.i32 = signext_from_bit(tmp.i32, S2[4 : 0].u32)',
+  VOP3Op.V_BFI_B32_E64: 'D0.u32 = ((S0.u32 & S1.u32) | (~S0.u32 & S2.u32))',
+  VOP3Op.V_FMA_F32_E64: 'D0.f32 = fma(S0.f32, S1.f32, S2.f32)',
+  VOP3Op.V_FMA_F64_E64: 'D0.f64 = fma(S0.f64, S1.f64, S2.f64)',
+  VOP3Op.V_LERP_U8_E64: 'tmp = ((S0.u32[31 : 24] + S1.u32[31 : 24] + S2.u32[24].u8) >> 1U << 24U);\ntmp += ((S0.u32[23 : 16] + S1.u32[23 : 16] + S2.u32[16].u8) >> 1U << 16U);\ntmp += ((S0.u32[15 : 8] + S1.u32[15 : 8] + S2.u32[8].u8) >> 1U << 8U);\ntmp += ((S0.u32[7 : 0] + S1.u32[7 : 0] + S2.u32[0].u8) >> 1U);\nD0.u32 = tmp.u32',
+  VOP3Op.V_ALIGNBIT_B32_E64: "D0.u32 = 32'U(({ S0.u32, S1.u32 } >> S2.u32[4 : 0]) & 0xffffffffLL)",
+  VOP3Op.V_ALIGNBYTE_B32_E64: "D0.u32 = 32'U(({ S0.u32, S1.u32 } >> (S2.u32[1 : 0] * 8U)) & 0xffffffffLL)",
+  VOP3Op.V_MIN3_F32_E64: 'D0.f32 = v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)',
+  VOP3Op.V_MIN3_I32_E64: 'D0.i32 = v_min_i32(v_min_i32(S0.i32, S1.i32), S2.i32)',
+  VOP3Op.V_MIN3_U32_E64: 'D0.u32 = v_min_u32(v_min_u32(S0.u32, S1.u32), S2.u32)',
+  VOP3Op.V_MAX3_F32_E64: 'D0.f32 = v_max_f32(v_max_f32(S0.f32, S1.f32), S2.f32)',
+  VOP3Op.V_MAX3_I32_E64: 'D0.i32 = v_max_i32(v_max_i32(S0.i32, S1.i32), S2.i32)',
+  VOP3Op.V_MAX3_U32_E64: 'D0.u32 = v_max_u32(v_max_u32(S0.u32, S1.u32), S2.u32)',
+  VOP3Op.V_MED3_F32_E64: "if (isNAN(64'F(S0.f32)) || isNAN(64'F(S1.f32)) || isNAN(64'F(S2.f32))) then\nD0.f32 = v_min3_f32(S0.f32, S1.f32, S2.f32)\nelsif v_max3_f32(S0.f32, S1.f32, S2.f32) == S0.f32 then\nD0.f32 = v_max_f32(S1.f32, S2.f32)\nelsif v_max3_f32(S0.f32, S1.f32, S2.f32) == S1.f32 then\nD0.f32 = v_max_f32(S0.f32, S2.f32)\nelse\nD0.f32 = v_max_f32(S0.f32, S1.f32)\nendif",
+  VOP3Op.V_MED3_I32_E64: 'if v_max3_i32(S0.i32, S1.i32, S2.i32) == S0.i32 then\nD0.i32 = v_max_i32(S1.i32, S2.i32)\nelsif v_max3_i32(S0.i32, S1.i32, S2.i32) == S1.i32 then\nD0.i32 = v_max_i32(S0.i32, S2.i32)\nelse\nD0.i32 = v_max_i32(S0.i32, S1.i32)\nendif',
+  VOP3Op.V_MED3_U32_E64: 'if v_max3_u32(S0.u32, S1.u32, S2.u32) == S0.u32 then\nD0.u32 = v_max_u32(S1.u32, S2.u32)\nelsif v_max3_u32(S0.u32, S1.u32, S2.u32) == S1.u32 then\nD0.u32 = v_max_u32(S0.u32, S2.u32)\nelse\nD0.u32 = v_max_u32(S0.u32, S1.u32)\nendif',
+  VOP3Op.V_SAD_U8_E64: "ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += 32'U(ABSDIFF(S0.u32[7 : 0], S1.u32[7 : 0]));\ntmp += 32'U(ABSDIFF(S0.u32[15 : 8], S1.u32[15 : 8]));\ntmp += 32'U(ABSDIFF(S0.u32[23 : 16], S1.u32[23 : 16]));\ntmp += 32'U(ABSDIFF(S0.u32[31 : 24], S1.u32[31 : 24]));\nD0.u32 = tmp",
+  VOP3Op.V_SAD_HI_U8_E64: "D0.u32 = (32'U(v_sad_u8(S0, S1, 0U)) << 16U) + S2.u32",
+  VOP3Op.V_SAD_U16_E64: 'ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += ABSDIFF(S0[15 : 0].u16, S1[15 : 0].u16);\ntmp += ABSDIFF(S0[31 : 16].u16, S1[31 : 16].u16);\nD0.u32 = tmp',
+  VOP3Op.V_SAD_U32_E64: 'ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\nD0.u32 = ABSDIFF(S0.u32, S1.u32) + S2.u32',
+  VOP3Op.V_CVT_PK_U8_F32_E64: "tmp = (S2.u32 & 32'U(~(0xff << (S1.u32[1 : 0].u32 * 8U))));\ntmp = (tmp | ((32'U(f32_to_u8(S0.f32)) & 255U) << (S1.u32[1 : 0].u32 * 8U)));\nD0.u32 = tmp",
+  VOP3Op.V_DIV_FIXUP_F32_E64: "sign_out = (sign(S1.f32) ^ sign(S2.f32));\nif isNAN(64'F(S2.f32)) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S2.f32)))\nelsif isNAN(64'F(S1.f32)) then\nD0.f32 = 32'F(cvtToQuietNAN(64'F(S1.f32)))\nelsif ((64'F(S1.f32) == 0.0) && (64'F(S2.f32) == 0.0)) then\n// 0/0\nD0.f32 = 32'F(0xffc00000)\nelsif ((64'F(abs(S1.f32)) == +INF) && (64'F(abs(S2.f32)) == +INF)) then\n// inf/inf\nD0.f32 = 32'F(0xffc00000)\nelsif ((64'F(S1.f32) == 0.0) || (64'F(abs(S2.f32)) == +INF)) then\n// x/0, or inf/y\nD0.f32 = sign_out ? -INF.f32 : +INF.f32\nelsif ((64'F(abs(S1.f32)) == +INF) || (64'F(S2.f32) == 0.0)) then\n// x/inf, 0/y\nD0.f32 = sign_out ? -0.0F : 0.0F\nelsif exponent(S2.f32) - exponent(S1.f32) < -150 then\nD0.f32 = sign_out ? -UNDERFLOW_F32 : UNDERFLOW_F32\nelsif exponent(S1.f32) == 255 then\nD0.f32 = sign_out ? -OVERFLOW_F32 : OVERFLOW_F32\nelse\nD0.f32 = sign_out ? -abs(S0.f32) : abs(S0.f32)\nendif",
+  VOP3Op.V_DIV_FIXUP_F64_E64: "sign_out = (sign(S1.f64) ^ sign(S2.f64));\nif isNAN(S2.f64) then\nD0.f64 = cvtToQuietNAN(S2.f64)\nelsif isNAN(S1.f64) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif ((S1.f64 == 0.0) && (S2.f64 == 0.0)) then\n// 0/0\nD0.f64 = 64'F(0xfff8000000000000LL)\nelsif ((abs(S1.f64) == +INF) && (abs(S2.f64) == +INF)) then\n// inf/inf\nD0.f64 = 64'F(0xfff8000000000000LL)\nelsif ((S1.f64 == 0.0) || (abs(S2.f64) == +INF)) then\n// x/0, or inf/y\nD0.f64 = sign_out ? -INF : +INF\nelsif ((abs(S1.f64) == +INF) || (S2.f64 == 0.0)) then\n// x/inf, 0/y\nD0.f64 = sign_out ? -0.0 : 0.0\nelsif exponent(S2.f64) - exponent(S1.f64) < -1075 then\nD0.f64 = sign_out ? -UNDERFLOW_F64 : UNDERFLOW_F64\nelsif exponent(S1.f64) == 2047 then\nD0.f64 = sign_out ? -OVERFLOW_F64 : OVERFLOW_F64\nelse\nD0.f64 = sign_out ? -abs(S0.f64) : abs(S0.f64)\nendif",
+  VOP3Op.V_DIV_FMAS_F32_E64: 'if VCC.u64[laneId] then\nD0.f32 = 2.0F ** 32 * fma(S0.f32, S1.f32, S2.f32)\nelse\nD0.f32 = fma(S0.f32, S1.f32, S2.f32)\nendif',
+  VOP3Op.V_DIV_FMAS_F64_E64: 'if VCC.u64[laneId] then\nD0.f64 = 2.0 ** 64 * fma(S0.f64, S1.f64, S2.f64)\nelse\nD0.f64 = fma(S0.f64, S1.f64, S2.f64)\nendif',
+  VOP3Op.V_MSAD_U8_E64: "ABSDIFF = lambda(x, y) (\nx > y ? x - y : y - x);\n// UNSIGNED comparison\ntmp = S2.u32;\ntmp += S1.u32[7 : 0] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[7 : 0], S1.u32[7 : 0]));\ntmp += S1.u32[15 : 8] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[15 : 8], S1.u32[15 : 8]));\ntmp += S1.u32[23 : 16] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[23 : 16], S1.u32[23 : 16]));\ntmp += S1.u32[31 : 24] == 8'0U ? 0U : 32'U(ABSDIFF(S0.u32[31 : 24], S1.u32[31 : 24]));\nD0.u32 = tmp",
+  VOP3Op.V_QSAD_PK_U16_U8_E64: "tmp[63 : 48] = 16'B(v_sad_u8(S0[55 : 24], S1[31 : 0], S2[63 : 48].u32));\ntmp[47 : 32] = 16'B(v_sad_u8(S0[47 : 16], S1[31 : 0], S2[47 : 32].u32));\ntmp[31 : 16] = 16'B(v_sad_u8(S0[39 : 8], S1[31 : 0], S2[31 : 16].u32));\ntmp[15 : 0] = 16'B(v_sad_u8(S0[31 : 0], S1[31 : 0], S2[15 : 0].u32));\nD0.b64 = tmp.b64",
+  VOP3Op.V_MQSAD_PK_U16_U8_E64: "tmp[63 : 48] = 16'B(v_msad_u8(S0[55 : 24], S1[31 : 0], S2[63 : 48].u32));\ntmp[47 : 32] = 16'B(v_msad_u8(S0[47 : 16], S1[31 : 0], S2[47 : 32].u32));\ntmp[31 : 16] = 16'B(v_msad_u8(S0[39 : 8], S1[31 : 0], S2[31 : 16].u32));\ntmp[15 : 0] = 16'B(v_msad_u8(S0[31 : 0], S1[31 : 0], S2[15 : 0].u32));\nD0.b64 = tmp.b64",
+  VOP3Op.V_MQSAD_U32_U8_E64: "tmp[127 : 96] = 32'B(v_msad_u8(S0[55 : 24], S1[31 : 0], S2[127 : 96].u32));\ntmp[95 : 64] = 32'B(v_msad_u8(S0[47 : 16], S1[31 : 0], S2[95 : 64].u32));\ntmp[63 : 32] = 32'B(v_msad_u8(S0[39 : 8], S1[31 : 0], S2[63 : 32].u32));\ntmp[31 : 0] = 32'B(v_msad_u8(S0[31 : 0], S1[31 : 0], S2[31 : 0].u32));\nD0.b128 = tmp.b128",
+  VOP3Op.V_MAD_LEGACY_F16_E64: "tmp = S0.f16 * S1.f16 + S2.f16;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
+  VOP3Op.V_MAD_LEGACY_U16_E64: "tmp = S0.u16 * S1.u16 + S2.u16;\nif OPSEL.u4[3] then\nD0 = { tmp.u16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.u16 }\nendif",
+  VOP3Op.V_MAD_LEGACY_I16_E64: "tmp = S0.i16 * S1.i16 + S2.i16;\nif OPSEL.u4[3] then\nD0 = { tmp.i16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.i16 }\nendif",
+  VOP3Op.V_PERM_B32_E64: "BYTE_PERMUTE = lambda(data, sel) (\ndeclare in : 8'B[8];\nfor i in 0 : 7 do\nin[i] = data[i * 8 + 7 : i * 8].b8\nendfor;\nif sel.u32 >= 13U then\nreturn 8'0xff\nelsif sel.u32 == 12U then\nreturn 8'0x0\nelsif sel.u32 == 11U then\nreturn in[7][7].b8 * 8'0xff\nelsif sel.u32 == 10U then\nreturn in[5][7].b8 * 8'0xff\nelsif sel.u32 == 9U then\nreturn in[3][7].b8 * 8'0xff\nelsif sel.u32 == 8U then\nreturn in[1][7].b8 * 8'0xff\nelse\nreturn in[sel]\nendif);\nD0[31 : 24] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[31 : 24]);\nD0[23 : 16] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[23 : 16]);\nD0[15 : 8] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[15 : 8]);\nD0[7 : 0] = BYTE_PERMUTE({ S0.u32, S1.u32 }, S2.u32[7 : 0])",
+  VOP3Op.V_FMA_LEGACY_F16_E64: "tmp = fma(S0.f16, S1.f16, S2.f16);\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
+  VOP3Op.V_DIV_FIXUP_LEGACY_F16_E64: "sign_out = (sign(S1.f16) ^ sign(S2.f16));\nif isNAN(64'F(S2.f16)) then\ntmp = cvtToQuietNAN(64'F(S2.f16))\nelsif isNAN(64'F(S1.f16)) then\ntmp = cvtToQuietNAN(64'F(S1.f16))\nelsif ((64'F(S1.f16) == 0.0) && (64'F(S2.f16) == 0.0)) then\n// 0/0\ntmp = 16'F(0xfe00)\nelsif ((64'F(abs(S1.f16)) == +INF) && (64'F(abs(S2.f16)) == +INF)) then\n// inf/inf\ntmp = 16'F(0xfe00)\nelsif ((64'F(S1.f16) == 0.0) || (64'F(abs(S2.f16)) == +INF)) then\n// x/0, or inf/y\ntmp = sign_out ? -INF : +INF\nelsif ((64'F(abs(S1.f16)) == +INF) || (64'F(S2.f16) == 0.0)) then\n// x/inf, 0/y\ntmp = sign_out ? -0.0 : 0.0\nelse\ntmp = sign_out ? -abs(S0.f16) : abs(S0.f16)\nendif;\nif OPSEL.u4[3] then\nD0 = { tmp.f16, D0[15 : 0] }\nelse\nD0 = { 16'0, tmp.f16 }\nendif",
+  VOP3Op.V_CVT_PKACCUM_U8_F32_E64: "byte = S1.u32[1 : 0];\nbit = byte.u32 * 8U;\nD0.u32[bit + 7U : bit] = 32'U(f32_to_u8(S0.f32))",
+  VOP3Op.V_MAD_U32_U16_E64: "D0.u32 = 32'U(S0.u16) * 32'U(S1.u16) + S2.u32",
+  VOP3Op.V_MAD_I32_I16_E64: "D0.i32 = 32'I(S0.i16) * 32'I(S1.i16) + S2.i32",
+  VOP3Op.V_XAD_U32_E64: 'D0.u32 = (S0.u32 ^ S1.u32) + S2.u32',
+  VOP3Op.V_MIN3_F16_E64: 'D0.f16 = v_min_f16(v_min_f16(S0.f16, S1.f16), S2.f16)',
+  VOP3Op.V_MIN3_I16_E64: 'D0.i16 = v_min_i16(v_min_i16(S0.i16, S1.i16), S2.i16)',
+  VOP3Op.V_MIN3_U16_E64: 'D0.u16 = v_min_u16(v_min_u16(S0.u16, S1.u16), S2.u16)',
+  VOP3Op.V_MAX3_F16_E64: 'D0.f16 = v_max_f16(v_max_f16(S0.f16, S1.f16), S2.f16)',
+  VOP3Op.V_MAX3_I16_E64: 'D0.i16 = v_max_i16(v_max_i16(S0.i16, S1.i16), S2.i16)',
+  VOP3Op.V_MAX3_U16_E64: 'D0.u16 = v_max_u16(v_max_u16(S0.u16, S1.u16), S2.u16)',
+  VOP3Op.V_MED3_F16_E64: "if (isNAN(64'F(S0.f16)) || isNAN(64'F(S1.f16)) || isNAN(64'F(S2.f16))) then\nD0.f16 = v_min3_f16(S0.f16, S1.f16, S2.f16)\nelsif v_max3_f16(S0.f16, S1.f16, S2.f16) == S0.f16 then\nD0.f16 = v_max_f16(S1.f16, S2.f16)\nelsif v_max3_f16(S0.f16, S1.f16, S2.f16) == S1.f16 then\nD0.f16 = v_max_f16(S0.f16, S2.f16)\nelse\nD0.f16 = v_max_f16(S0.f16, S1.f16)\nendif",
+  VOP3Op.V_MED3_I16_E64: 'if v_max3_i16(S0.i16, S1.i16, S2.i16) == S0.i16 then\nD0.i16 = v_max_i16(S1.i16, S2.i16)\nelsif v_max3_i16(S0.i16, S1.i16, S2.i16) == S1.i16 then\nD0.i16 = v_max_i16(S0.i16, S2.i16)\nelse\nD0.i16 = v_max_i16(S0.i16, S1.i16)\nendif',
+  VOP3Op.V_MED3_U16_E64: 'if v_max3_u16(S0.u16, S1.u16, S2.u16) == S0.u16 then\nD0.u16 = v_max_u16(S1.u16, S2.u16)\nelsif v_max3_u16(S0.u16, S1.u16, S2.u16) == S1.u16 then\nD0.u16 = v_max_u16(S0.u16, S2.u16)\nelse\nD0.u16 = v_max_u16(S0.u16, S1.u16)\nendif',
+  VOP3Op.V_LSHL_ADD_U32_E64: 'D0.u32 = (S0.u32 << S1.u32[4 : 0].u32) + S2.u32',
+  VOP3Op.V_ADD_LSHL_U32_E64: 'D0.u32 = ((S0.u32 + S1.u32) << S2.u32[4 : 0].u32)',
+  VOP3Op.V_ADD3_U32_E64: 'D0.u32 = S0.u32 + S1.u32 + S2.u32',
+  VOP3Op.V_LSHL_OR_B32: 'D0.u32 = ((S0.u32 << S1.u32[4 : 0].u32) | S2.u32)',
+  VOP3Op.V_AND_OR_B32: 'D0.u32 = ((S0.u32 & S1.u32) | S2.u32)',
+  VOP3Op.V_OR3_B32: 'D0.u32 = (S0.u32 | S1.u32 | S2.u32)',
+  VOP3Op.V_MAD_F16: 'D0.f16 = S0.f16 * S1.f16 + S2.f16',
+  VOP3Op.V_MAD_U16: 'D0.u16 = S0.u16 * S1.u16 + S2.u16',
+  VOP3Op.V_MAD_I16: 'D0.i16 = S0.i16 * S1.i16 + S2.i16',
+  VOP3Op.V_FMA_F16: 'D0.f16 = fma(S0.f16, S1.f16, S2.f16)',
+  VOP3Op.V_DIV_FIXUP_F16: "sign_out = (sign(S1.f16) ^ sign(S2.f16));\nif isNAN(64'F(S2.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S2.f16)))\nelsif isNAN(64'F(S1.f16)) then\nD0.f16 = 16'F(cvtToQuietNAN(64'F(S1.f16)))\nelsif ((64'F(S1.f16) == 0.0) && (64'F(S2.f16) == 0.0)) then\n// 0/0\nD0.f16 = 16'F(0xfe00)\nelsif ((64'F(abs(S1.f16)) == +INF) && (64'F(abs(S2.f16)) == +INF)) then\n// inf/inf\nD0.f16 = 16'F(0xfe00)\nelsif ((64'F(S1.f16) == 0.0) || (64'F(abs(S2.f16)) == +INF)) then\n// x/0, or inf/y\nD0.f16 = sign_out ? -INF.f16 : +INF.f16\nelsif ((64'F(abs(S1.f16)) == +INF) || (64'F(S2.f16) == 0.0)) then\n// x/inf, 0/y\nD0.f16 = sign_out ? -16'0.0 : 16'0.0\nelse\nD0.f16 = sign_out ? -abs(S0.f16) : abs(S0.f16)\nendif",
+  VOP3Op.V_LSHL_ADD_U64: 'D0.u64 = (S0.u64 << S1.u32[2 : 0].u32) + S2.u64',
+  VOP3Op.V_BITOP3_B16: "TTBL = { INST.OMOD[1 : 0], INST.ABS[2 : 0], INST.NEG[2 : 0] };\ntmp = 16'0U;\ntmp = (tmp | (32'I(TTBL.b32 & 0x1) != 0 ? 16'U(~S0.b16 & ~S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x2) != 0 ? 16'U(~S0.b16 & ~S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x4) != 0 ? 16'U(~S0.b16 & S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x8) != 0 ? 16'U(~S0.b16 & S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x10) != 0 ? 16'U(S0.b16 & ~S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x20) != 0 ? 16'U(S0.b16 & ~S1.b16 & S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x40) != 0 ? 16'U(S0.b16 & S1.b16 & ~S2.b16) : 16'0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x80) != 0 ? 16'U(S0.b16 & S1.b16 & S2.b16) : 16'0U));\nD.b16 = tmp.b16",
+  VOP3Op.V_BITOP3_B32: "TTBL = { INST.OMOD[1 : 0], INST.ABS[2 : 0], INST.NEG[2 : 0] };\ntmp = 0U;\ntmp = (tmp | (32'I(TTBL.b32 & 0x1) != 0 ? 32'U(~S0.b32 & ~S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x2) != 0 ? 32'U(~S0.b32 & ~S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x4) != 0 ? 32'U(~S0.b32 & S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x8) != 0 ? 32'U(~S0.b32 & S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x10) != 0 ? 32'U(S0.b32 & ~S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x20) != 0 ? 32'U(S0.b32 & ~S1.b32 & S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x40) != 0 ? 32'U(S0.b32 & S1.b32 & ~S2.b32) : 0U));\ntmp = (tmp | (32'I(TTBL.b32 & 0x80) != 0 ? 32'U(S0.b32 & S1.b32 & S2.b32) : 0U));\nD.b32 = tmp.b32",
+  VOP3Op.V_CVT_SCALEF32_PK_FP8_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_fp8_scale(S0.f32, scale.u8);\ntmp1 = f32_to_fp8_scale(S1.f32, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_BF8_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_bf8_scale(S0.f32, scale.u8);\ntmp1 = f32_to_bf8_scale(S1.f32, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_FP8_F32: "scale = 32'U(exponent(S2.f32));\ntmp = f32_to_fp8_sr_scale(S0.f32, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_BF8_F32: "scale = 32'U(exponent(S2.f32));\ntmp = f32_to_bf8_sr_scale(S0.f32, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_F32_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_f32_scale(src[7 : 0].fp8, scale.u8);\ntmp1 = fp8_to_f32_scale(src[15 : 8].fp8, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_PK_F32_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_f32_scale(src[7 : 0].bf8, scale.u8);\ntmp1 = bf8_to_f32_scale(src[15 : 8].bf8, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_F32_FP8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].fp8;\ntmp = fp8_to_f32_scale(src, scale.u8);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_SCALEF32_F32_BF8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].bf8;\ntmp = bf8_to_f32_scale(src, scale.u8);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_SCALEF32_PK_FP4_F32: "scale = 32'U(exponent(S2.f32));\ntmp0 = f32_to_fp4_scale(S0.f32, scale.u8);\ntmp1 = f32_to_fp4_scale(S1.f32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_PK_FP4_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = f32_to_fp4_sr_scale(S0[31 : 0].f32, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = f32_to_fp4_sr_scale(S0[63 : 32].f32, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_F32_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_f32_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_f32_scale(src[7 : 4].fp4, scale.u8);\nD0[31 : 0].f32 = tmp0;\nD0[63 : 32].f32 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_PK_FP8_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_fp8_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_fp8_scale(S0[31 : 16].f16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_BF8_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_bf8_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_bf8_scale(S0[31 : 16].f16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_FP8_F16: "scale = 32'U(exponent(S2.f32));\ntmp = f16_to_fp8_sr_scale(S0.f16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_BF8_F16: "scale = 32'U(exponent(S2.f32));\ntmp = f16_to_bf8_sr_scale(S0.f16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_FP8_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_fp8_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_fp8_scale(S0[31 : 16].bf16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_BF8_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_bf8_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_bf8_scale(S0[31 : 16].bf16, scale.u8);\ndstword = OPSEL[3].i32 * 16;\nVGPR[laneId][VDST.u32][dstword + 15 : dstword].b16 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_FP8_BF16: "scale = 32'U(exponent(S2.f32));\ntmp = bf16_to_fp8_sr_scale(S0.bf16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].fp8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_BF8_BF16: "scale = 32'U(exponent(S2.f32));\ntmp = bf16_to_bf8_sr_scale(S0.bf16, S1.u32, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].bf8 = tmp;\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_F16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_f16_scale(src[7 : 0].fp8, scale.u8);\ntmp1 = fp8_to_f16_scale(src[15 : 8].fp8, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_PK_F16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_f16_scale(src[7 : 0].bf8, scale.u8);\ntmp1 = bf8_to_f16_scale(src[15 : 8].bf8, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_F16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].fp8;\ntmp = fp8_to_f16_scale(src, scale.u8);\n// OPSEL[3] controls destination hi/lo\nD0 = tmp.b32",
+  VOP3Op.V_CVT_SCALEF32_F16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].bf8;\ntmp = bf8_to_f16_scale(src, scale.u8);\n// OPSEL[3] controls destination hi/lo\nD0 = tmp.b32",
+  VOP3Op.V_CVT_SCALEF32_PK_FP4_F16: "scale = 32'U(exponent(S1.f32));\ntmp0 = f16_to_fp4_scale(S0[15 : 0].f16, scale.u8);\ntmp1 = f16_to_fp4_scale(S0[31 : 16].f16, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_FP4_BF16: "scale = 32'U(exponent(S1.f32));\ntmp0 = bf16_to_fp4_scale(S0[15 : 0].bf16, scale.u8);\ntmp1 = bf16_to_fp4_scale(S0[31 : 16].bf16, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_PK_FP4_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = f16_to_fp4_sr_scale(S0[15 : 0].f16, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = f16_to_fp4_sr_scale(S0[31 : 16].f16, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_SR_PK_FP4_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ntmp0 = bf16_to_fp4_sr_scale(S0[15 : 0].bf16, randomVal, scale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32));\ntmp1 = bf16_to_fp4_sr_scale(S0[31 : 16].bf16, randomVal, scale.u8);\ndstbyte = OPSEL[3 : 2].i32 * 8;\nVGPR[laneId][VDST.u32][dstbyte + 7 : dstbyte].b8 = { tmp1, tmp0 };\n// Other destination bits are preserved",
+  VOP3Op.V_CVT_SCALEF32_PK_F16_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_f16_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_f16_scale(src[7 : 4].fp4, scale.u8);\nD0[15 : 0].f16 = tmp0;\nD0[31 : 16].f16 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_PK_BF16_FP4: "scale = 32'U(exponent(S1.f32));\nsrcbyte = OPSEL[1 : 0].i32 * 8;\nsrc = VGPR[laneId][SRC0.u32][srcbyte + 7 : srcbyte].b8;\ntmp0 = fp4_to_bf16_scale(src[3 : 0].fp4, scale.u8);\ntmp1 = fp4_to_bf16_scale(src[7 : 4].fp4, scale.u8);\nD0[15 : 0].bf16 = tmp0;\nD0[31 : 16].bf16 = tmp1",
+  VOP3Op.V_CVT_SCALEF32_2XPK16_FP6_F32: "scale = 32'U(exponent(S2.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 15 do\ndOffset = pass * 12;\nsOffset = pass * 32;\n// Note that S0 and S1 inputs are interleaved in the packed result.\ntmp[dOffset + 5 : dOffset].fp6 = f32_to_fp6_scale(S0[sOffset + 31 : sOffset].f32, scale.u8);\ntmp[dOffset + 11 : dOffset + 6].fp6 = f32_to_fp6_scale(S1[sOffset + 31 : sOffset].f32, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_2XPK16_BF6_F32: "scale = 32'U(exponent(S2.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 15 do\ndOffset = pass * 12;\nsOffset = pass * 32;\n// Note that S0 and S1 inputs are interleaved in the packed result.\ntmp[dOffset + 5 : dOffset].bf6 = f32_to_bf6_scale(S0[sOffset + 31 : sOffset].f32, scale.u8);\ntmp[dOffset + 11 : dOffset + 6].bf6 = f32_to_bf6_scale(S1[sOffset + 31 : sOffset].f32, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_FP6_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 32;\ntmp[dOffset + 5 : dOffset].fp6 = f32_to_fp6_sr_scale(S0[sOffset + 31 : sOffset].f32, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_BF6_F32: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 32;\ntmp[dOffset + 5 : dOffset].bf6 = f32_to_bf6_sr_scale(S0[sOffset + 31 : sOffset].f32, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_PK32_F32_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 1024'B;\nfor pass in 0 : 31 do\ndOffset = pass * 32;\nsOffset = pass * 6;\ntmp[dOffset + 31 : dOffset].f32 = fp6_to_f32_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[1023 : 0] = tmp.b1024",
+  VOP3Op.V_CVT_SCALEF32_PK32_F32_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 1024'B;\nfor pass in 0 : 31 do\ndOffset = pass * 32;\nsOffset = pass * 6;\ntmp[dOffset + 31 : dOffset].f32 = bf6_to_f32_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[1023 : 0] = tmp.b1024",
+  VOP3Op.V_CVT_SCALEF32_PK32_FP6_F16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = f16_to_fp6_scale(S0[sOffset + 15 : sOffset].f16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_PK32_FP6_BF16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = bf16_to_fp6_scale(S0[sOffset + 15 : sOffset].bf16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_PK32_BF6_F16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = f16_to_bf6_scale(S0[sOffset + 15 : sOffset].f16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_PK32_BF6_BF16: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = bf16_to_bf6_scale(S0[sOffset + 15 : sOffset].bf16, scale.u8)\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_FP6_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = f16_to_fp6_sr_scale(S0[sOffset + 15 : sOffset].f16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_FP6_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].fp6 = bf16_to_fp6_sr_scale(S0[sOffset + 15 : sOffset].bf16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_BF6_F16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = f16_to_bf6_sr_scale(S0[sOffset + 15 : sOffset].f16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_SR_PK32_BF6_BF16: "scale = 32'U(exponent(S2.f32));\nrandomVal = S1.u32;\ndeclare tmp : 192'B;\nfor pass in 0 : 31 do\ndOffset = pass * 6;\nsOffset = pass * 16;\ntmp[dOffset + 5 : dOffset].bf6 = bf16_to_bf6_sr_scale(S0[sOffset + 15 : sOffset].bf16, randomVal,\nscale.u8);\nrandomVal = 32'U(v_prng_b32(randomVal.b32))\nendfor;\nD0[191 : 0] = tmp.b192",
+  VOP3Op.V_CVT_SCALEF32_PK32_F16_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].f16 = fp6_to_f16_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
+  VOP3Op.V_CVT_SCALEF32_PK32_BF16_FP6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].bf16 = fp6_to_bf16_scale(S0[sOffset + 5 : sOffset].fp6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
+  VOP3Op.V_CVT_SCALEF32_PK32_F16_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].f16 = bf6_to_f16_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
+  VOP3Op.V_CVT_SCALEF32_PK32_BF16_BF6: "scale = 32'U(exponent(S1.f32));\ndeclare tmp : 512'B;\nfor pass in 0 : 31 do\ndOffset = pass * 16;\nsOffset = pass * 6;\ntmp[dOffset + 15 : dOffset].bf16 = bf6_to_bf16_scale(S0[sOffset + 5 : sOffset].bf6, scale.u8)\nendfor;\nD0[511 : 0] = tmp.b512",
+  VOP3Op.V_ASHR_PK_I8_I32: "SAT8 = lambda(n) (\nif n <= -128 then\nreturn 8'0x80\nelsif n >= 127 then\nreturn 8'0x7f\nelse\nreturn n[7 : 0].b8\nendif);\ndeclare tmp : 16'B;\ntmp[7 : 0] = SAT8(S0.i32 >> S2[4 : 0].u32);\ntmp[15 : 8] = SAT8(S1.i32 >> S2[4 : 0].u32);\nD0[15 : 0] = tmp",
+  VOP3Op.V_ASHR_PK_U8_I32: "SAT8 = lambda(n) (\nif n <= 0 then\nreturn 8'0x0\nelsif n >= 255 then\nreturn 8'0xff\nelse\nreturn n[7 : 0].b8\nendif);\ndeclare tmp : 16'B;\ntmp[7 : 0] = SAT8(S0.i32 >> S2[4 : 0].u32);\ntmp[15 : 8] = SAT8(S1.i32 >> S2[4 : 0].u32);\nD0[15 : 0] = tmp",
+  VOP3Op.V_CVT_PK_F16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ntmp[15 : 0].f16 = f32_to_f16(S0.f32);\ntmp[31 : 16].f16 = f32_to_f16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode',
+  VOP3Op.V_CVT_PK_BF16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ntmp[15 : 0].bf16 = f32_to_bf16(S0.f32);\ntmp[31 : 16].bf16 = f32_to_bf16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode',
+  VOP3Op.V_CVT_SCALEF32_PK_BF16_FP8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = fp8_to_bf16_scale(src[7 : 0].fp8, scale);\ntmp1 = fp8_to_bf16_scale(src[15 : 8].fp8, scale);\nD0[15 : 0].bf16 = tmp0.bf16;\nD0[31 : 16].bf16 = tmp1.bf16",
+  VOP3Op.V_CVT_SCALEF32_PK_BF16_BF8: "scale = 32'U(exponent(S1.f32));\nsrcword = OPSEL[0].i32 * 16;\nsrc = VGPR[laneId][SRC0.u32][srcword + 15 : srcword].b16;\ntmp0 = bf8_to_bf16_scale(src[7 : 0].bf8, scale);\ntmp1 = bf8_to_bf16_scale(src[15 : 8].bf8, scale);\nD0[15 : 0].bf16 = tmp0.bf16;\nD0[31 : 16].bf16 = tmp1.bf16",
+  VOP3Op.V_ADD_F64: 'D0.f64 = S0.f64 + S1.f64',
+  VOP3Op.V_MUL_F64: 'D0.f64 = S0.f64 * S1.f64',
+  VOP3Op.V_MIN_F64: "if (WAVE_MODE.IEEE && isSignalNAN(S0.f64)) then\nD0.f64 = cvtToQuietNAN(S0.f64)\nelsif (WAVE_MODE.IEEE && isSignalNAN(S1.f64)) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif isNAN(S0.f64) then\nD0.f64 = S1.f64\nelsif isNAN(S1.f64) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == +0.0) && (S1.f64 == -0.0)) then\nD0.f64 = S1.f64\nelsif ((S0.f64 == -0.0) && (S1.f64 == +0.0)) then\nD0.f64 = S0.f64\nelse\n// Note: there's no IEEE case here like there is for V_MAX_F64.\nD0.f64 = S0.f64 < S1.f64 ? S0.f64 : S1.f64\nendif",
+  VOP3Op.V_MAX_F64: 'if (WAVE_MODE.IEEE && isSignalNAN(S0.f64)) then\nD0.f64 = cvtToQuietNAN(S0.f64)\nelsif (WAVE_MODE.IEEE && isSignalNAN(S1.f64)) then\nD0.f64 = cvtToQuietNAN(S1.f64)\nelsif isNAN(S0.f64) then\nD0.f64 = S1.f64\nelsif isNAN(S1.f64) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == +0.0) && (S1.f64 == -0.0)) then\nD0.f64 = S0.f64\nelsif ((S0.f64 == -0.0) && (S1.f64 == +0.0)) then\nD0.f64 = S1.f64\nelsif WAVE_MODE.IEEE then\nD0.f64 = S0.f64 >= S1.f64 ? S0.f64 : S1.f64\nelse\nD0.f64 = S0.f64 > S1.f64 ? S0.f64 : S1.f64\nendif',
+  VOP3Op.V_LDEXP_F64: 'D0.f64 = S0.f64 * 2.0 ** S1.i32',
+  VOP3Op.V_MUL_LO_U32: 'D0.u32 = S0.u32 * S1.u32',
+  VOP3Op.V_MUL_HI_U32: "D0.u32 = 32'U((64'U(S0.u32) * 64'U(S1.u32)) >> 32U)",
+  VOP3Op.V_MUL_HI_I32: "D0.i32 = 32'I((64'I(S0.i32) * 64'I(S1.i32)) >> 32U)",
+  VOP3Op.V_LDEXP_F32: 'D0.f32 = S0.f32 * 2.0F ** S1.i32',
+  VOP3Op.V_READLANE_B32: 'lane = S1.u32[5 : 0];\n// Lane select\nD0.b32 = VGPR[lane][SRC0.u32]',
+  VOP3Op.V_WRITELANE_B32: 'lane = S1.u32[5 : 0];\n// Lane select\nVGPR[lane][VDST.u32] = S0.b32',
+  VOP3Op.V_BCNT_U32_B32: "tmp = S1.u32;\nfor i in 0 : 31 do\ntmp += S0[i].u32;\n// count i'th bit\nendfor;\nD0.u32 = tmp",
+  VOP3Op.V_MBCNT_LO_U32_B32: "ThreadMask = (1LL << laneId.u32) - 1LL;\nMaskedValue = (S0.u32 & ThreadMask[31 : 0].u32);\ntmp = S1.u32;\nfor i in 0 : 31 do\ntmp += MaskedValue[i] == 1'1U ? 1U : 0U\nendfor;\nD0.u32 = tmp",
+  VOP3Op.V_MBCNT_HI_U32_B32: "ThreadMask = (1LL << laneId.u32) - 1LL;\nMaskedValue = (S0.u32 & ThreadMask[63 : 32].u32);\ntmp = S1.u32;\nfor i in 0 : 31 do\ntmp += MaskedValue[i] == 1'1U ? 1U : 0U\nendfor;\nD0.u32 = tmp",
+  VOP3Op.V_LSHLREV_B64: 'D0.u64 = (S1.u64 << S0[5 : 0].u32)',
+  VOP3Op.V_LSHRREV_B64: 'D0.u64 = (S1.u64 >> S0[5 : 0].u32)',
+  VOP3Op.V_ASHRREV_I64: 'D0.i64 = (S1.i64 >> S0[5 : 0].u32)',
+  VOP3Op.V_TRIG_PREOP_F64: "shift = 32'I(S1[4 : 0].u32) * 53;\nif exponent(S0.f64) > 1077 then\nshift += exponent(S0.f64) - 1077\nendif;\n// (2.0/PI) == 0.{b_1200, b_1199, b_1198, ..., b_1, b_0}\n// b_1200 is the MSB of the fractional part of 2.0/PI\n// Left shift operation indicates which bits are brought\n// into the whole part of the number.\n// Only whole part of result is kept.\nresult = 64'F((1201'B(2.0 / PI)[1200 : 0] << shift.u32) & 1201'0x1fffffffffffff);\nscale = -53 - shift;\nif exponent(S0.f64) >= 1968 then\nscale += 128\nendif;\nD0.f64 = ldexp(result, scale)",
+  VOP3Op.V_BFM_B32: 'D0.u32 = (((1U << S0[4 : 0].u32) - 1U) << S1[4 : 0].u32)',
+  VOP3Op.V_CVT_PKNORM_I16_F32: "declare tmp : 32'B;\ntmp[15 : 0].i16 = f32_to_snorm(S0.f32);\ntmp[31 : 16].i16 = f32_to_snorm(S1.f32);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_PKNORM_U16_F32: "declare tmp : 32'B;\ntmp[15 : 0].u16 = f32_to_unorm(S0.f32);\ntmp[31 : 16].u16 = f32_to_unorm(S1.f32);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_PKRTZ_F16_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_TOWARD_ZERO;\ntmp[15 : 0].f16 = f32_to_f16(S0.f32);\ntmp[31 : 16].f16 = f32_to_f16(S1.f32);\nD0 = tmp.b32;\nROUND_MODE = prev_mode;\n// Round-toward-zero regardless of current round mode setting in hardware.',
+  VOP3Op.V_CVT_PK_U16_U32: "declare tmp : 32'B;\ntmp[15 : 0].u16 = u32_to_u16(S0.u32);\ntmp[31 : 16].u16 = u32_to_u16(S1.u32);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_PK_I16_I32: "declare tmp : 32'B;\ntmp[15 : 0].i16 = i32_to_i16(S0.i32);\ntmp[31 : 16].i16 = i32_to_i16(S1.i32);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_PKNORM_I16_F16: "declare tmp : 32'B;\ntmp[15 : 0].i16 = f16_to_snorm(S0.f16);\ntmp[31 : 16].i16 = f16_to_snorm(S1.f16);\nD0 = tmp.b32",
+  VOP3Op.V_CVT_PKNORM_U16_F16: "declare tmp : 32'B;\ntmp[15 : 0].u16 = f16_to_unorm(S0.f16);\ntmp[31 : 16].u16 = f16_to_unorm(S1.f16);\nD0 = tmp.b32",
+  VOP3Op.V_ADD_I32: 'D0.i32 = S0.i32 + S1.i32',
+  VOP3Op.V_SUB_I32: 'D0.i32 = S0.i32 - S1.i32',
+  VOP3Op.V_ADD_I16: 'D0.i16 = S0.i16 + S1.i16',
+  VOP3Op.V_SUB_I16: 'D0.i16 = S0.i16 - S1.i16',
+  VOP3Op.V_PACK_B32_F16: 'D0[31 : 16].f16 = S1.f16;\nD0[15 : 0].f16 = S0.f16',
+  VOP3Op.V_MUL_LEGACY_F32: "if ((64'F(S0.f32) == 0.0) || (64'F(S1.f32) == 0.0)) then\n// DX9 rules, 0.0 * x = 0.0\nD0.f32 = 0.0F\nelse\nD0.f32 = S0.f32 * S1.f32\nendif",
+  VOP3Op.V_CVT_PK_FP8_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u32 == 0U then\nVGPR[laneId][VDST.u32][15 : 0].b16 = { f32_to_fp8(S1.f32), f32_to_fp8(S0.f32) };\n// D0[31:16] are preserved\nelse\nVGPR[laneId][VDST.u32][31 : 16].b16 = { f32_to_fp8(S1.f32), f32_to_fp8(S0.f32) };\n// D0[15:0] are preserved\nendif;\nROUND_MODE = prev_mode',
+  VOP3Op.V_CVT_PK_BF8_F32: 'prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u32 == 0U then\nVGPR[laneId][VDST.u32][15 : 0].b16 = { f32_to_bf8(S1.f32), f32_to_bf8(S0.f32) };\n// D0[31:16] are preserved\nelse\nVGPR[laneId][VDST.u32][31 : 16].b16 = { f32_to_bf8(S1.f32), f32_to_bf8(S0.f32) };\n// D0[15:0] are preserved\nendif;\nROUND_MODE = prev_mode',
+  VOP3Op.V_CVT_SR_FP8_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ns = sign(S0.f32);\ne = exponent(S0.f32);\nm = 23'U(32'U(23'B(mantissa(S0.f32))) + S1[31 : 12].u32);\ntmp = float32(s, e, m);\n// Add stochastic value to mantissa, wrap around on overflow\nif OPSEL[3 : 2].u2 == 2'0U then\nVGPR[laneId][VDST.u32][7 : 0].fp8 = f32_to_fp8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'1U then\nVGPR[laneId][VDST.u32][15 : 8].fp8 = f32_to_fp8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'2U then\nVGPR[laneId][VDST.u32][23 : 16].fp8 = f32_to_fp8(tmp.f32)\nelse\nVGPR[laneId][VDST.u32][31 : 24].fp8 = f32_to_fp8(tmp.f32)\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
+  VOP3Op.V_CVT_SR_BF8_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\ns = sign(S0.f32);\ne = exponent(S0.f32);\nm = 23'U(32'U(23'B(mantissa(S0.f32))) + S1[31 : 11].u32);\ntmp = float32(s, e, m);\n// Add stochastic value to mantissa, wrap around on overflow\nif OPSEL[3 : 2].u2 == 2'0U then\nVGPR[laneId][VDST.u32][7 : 0].bf8 = f32_to_bf8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'1U then\nVGPR[laneId][VDST.u32][15 : 8].bf8 = f32_to_bf8(tmp.f32)\nelsif OPSEL[3 : 2].u2 == 2'2U then\nVGPR[laneId][VDST.u32][23 : 16].bf8 = f32_to_bf8(tmp.f32)\nelse\nVGPR[laneId][VDST.u32][31 : 24].bf8 = f32_to_bf8(tmp.f32)\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
+  VOP3Op.V_CVT_SR_F16_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u2 == 2'0U then\nVGPR[laneId][VDST.u32][15 : 0].f16 = 16'F(f32_to_f16_SR(S0.f32, S1.u32))\nelse\nVGPR[laneId][VDST.u32][31 : 16].f16 = 16'F(f32_to_f16_sr(S0.f32, S1.u32))\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
+  VOP3Op.V_CVT_SR_BF16_F32: "prev_mode = ROUND_MODE;\nROUND_MODE = ROUND_NEAREST_EVEN;\nif OPSEL[3].u2 == 2'0U then\nVGPR[laneId][VDST.u32][15 : 0].bf16 = 16'BF(f32_to_bf16_SR(S0.f32, S1.u32))\nelse\nVGPR[laneId][VDST.u32][31 : 16].bf16 = 16'BF(f32_to_bf16_sr(S0.f32, S1.u32))\nendif;\n// Unwritten bytes of D are preserved.\nROUND_MODE = prev_mode",
+  VOP3Op.V_MINIMUM3_F32: "D0.f32 = 32'F(v_minimum_f32(v_minimum_f32(S0.f32, S1.f32), S2.f32))",
+  VOP3Op.V_MAXIMUM3_F32: "D0.f32 = 32'F(v_maximum_f32(v_maximum_f32(S0.f32, S1.f32), S2.f32))",
   VOP3POp.V_PK_MAD_I16: "declare tmp : 32'B;\ntmp[15 : 0].i16 = S0[15 : 0].i16 * S1[15 : 0].i16 + S2[15 : 0].i16;\ntmp[31 : 16].i16 = S0[31 : 16].i16 * S1[31 : 16].i16 + S2[31 : 16].i16;\nD0.b32 = tmp",
   VOP3POp.V_PK_MUL_LO_U16: 'tmp[31 : 16].u16 = S0[31 : 16].u16 * S1[31 : 16].u16;\ntmp[15 : 0].u16 = S0[15 : 0].u16 * S1[15 : 0].u16;\nD0.b32 = tmp.b32',
   VOP3POp.V_PK_ADD_I16: "declare tmp : 32'B;\ntmp[15 : 0].i16 = S0[15 : 0].i16 + S1[15 : 0].i16;\ntmp[31 : 16].i16 = S0[31 : 16].i16 + S1[31 : 16].i16;\nD0.b32 = tmp",
@@ -1246,6 +1236,16 @@ PCODE = {
   VOP3POp.V_SMFMAC_F32_32X32X32_BF8_FP8: 'D = A (sparse 32x32) * B (32x32) + D (32x32)',
   VOP3POp.V_SMFMAC_F32_32X32X32_FP8_BF8: 'D = A (sparse 32x32) * B (32x32) + D (32x32)',
   VOP3POp.V_SMFMAC_F32_32X32X32_FP8_FP8: 'D = A (sparse 32x32) * B (32x32) + D (32x32)',
+  VOP3SDOp.V_ADD_CO_U32: "tmp = 64'U(S0.u32) + 64'U(S1.u32);\nVCC.u64[laneId] = tmp >= 0x100000000ULL ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_ADDC_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_SUB_CO_U32: "tmp = S0.u32 - S1.u32;\nVCC.u64[laneId] = S1.u32 > S0.u32 ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_SUBREV_CO_U32: "tmp = S1.u32 - S0.u32;\nVCC.u64[laneId] = S0.u32 > S1.u32 ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_ADDC_CO_U32: "tmp = 64'U(S0.u32) + 64'U(S1.u32) + VCC.u64[laneId].u64;\nVCC.u64[laneId] = tmp >= 0x100000000ULL ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_ADDC_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_SUBB_CO_U32: "tmp = S0.u32 - S1.u32 - VCC.u64[laneId].u32;\nVCC.u64[laneId] = 64'U(S1.u32) + VCC.u64[laneId].u64 > 64'U(S0.u32) ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_SUBBREV_CO_U32: "tmp = S1.u32 - S0.u32 - VCC.u64[laneId].u32;\nVCC.u64[laneId] = 64'U(S0.u32) + VCC.u64[laneId].u64 > 64'U(S1.u32) ? 1'1U : 1'0U;\n// VCC is an UNSIGNED overflow/carry-out for V_SUBB_CO_U32.\nD0.u32 = tmp.u32",
+  VOP3SDOp.V_DIV_SCALE_F32: "VCC = 0x0LL;\nif ((64'F(S2.f32) == 0.0) || (64'F(S1.f32) == 0.0)) then\nD0.f32 = NAN.f32\nelsif exponent(S2.f32) - exponent(S1.f32) >= 96 then\n// N/D near MAX_FLOAT_F32\nVCC = 0x1LL;\nif S0.f32 == S1.f32 then\n// Only scale the denominator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif S1.f32 == DENORM.f32 then\nD0.f32 = ldexp(S0.f32, 64)\nelsif ((1.0 / 64'F(S1.f32) == DENORM.f64) && (S2.f32 / S1.f32 == DENORM.f32)) then\nVCC = 0x1LL;\nif S0.f32 == S1.f32 then\n// Only scale the denominator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif 1.0 / 64'F(S1.f32) == DENORM.f64 then\nD0.f32 = ldexp(S0.f32, -64)\nelsif S2.f32 / S1.f32 == DENORM.f32 then\nVCC = 0x1LL;\nif S0.f32 == S2.f32 then\n// Only scale the numerator\nD0.f32 = ldexp(S0.f32, 64)\nendif\nelsif exponent(S2.f32) <= 23 then\n// Numerator is tiny\nD0.f32 = ldexp(S0.f32, 64)\nendif",
+  VOP3SDOp.V_DIV_SCALE_F64: 'VCC = 0x0LL;\nif ((S2.f64 == 0.0) || (S1.f64 == 0.0)) then\nD0.f64 = NAN.f64\nelsif exponent(S2.f64) - exponent(S1.f64) >= 768 then\n// N/D near MAX_FLOAT_F64\nVCC = 0x1LL;\nif S0.f64 == S1.f64 then\n// Only scale the denominator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif S1.f64 == DENORM.f64 then\nD0.f64 = ldexp(S0.f64, 128)\nelsif ((1.0 / S1.f64 == DENORM.f64) && (S2.f64 / S1.f64 == DENORM.f64)) then\nVCC = 0x1LL;\nif S0.f64 == S1.f64 then\n// Only scale the denominator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif 1.0 / S1.f64 == DENORM.f64 then\nD0.f64 = ldexp(S0.f64, -128)\nelsif S2.f64 / S1.f64 == DENORM.f64 then\nVCC = 0x1LL;\nif S0.f64 == S2.f64 then\n// Only scale the numerator\nD0.f64 = ldexp(S0.f64, 128)\nendif\nelsif exponent(S2.f64) <= 53 then\n// Numerator is tiny\nD0.f64 = ldexp(S0.f64, 128)\nendif',
+  VOP3SDOp.V_MAD_U64_U32: "{ D1.u1, D0.u64 } = 65'B(65'U(S0.u32) * 65'U(S1.u32) + 65'U(S2.u64))",
+  VOP3SDOp.V_MAD_I64_I32: "{ D1.i1, D0.i64 } = 65'B(65'I(S0.i32) * 65'I(S1.i32) + 65'I(S2.i64))",
   VOPCOp.V_CMP_CLASS_F32_E32: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",
   VOPCOp.V_CMPX_CLASS_F32_E32: "declare result : 1'U;\nif isSignalNAN(64'F(S0.f32)) then\nresult = S1.u32[0]\nelsif isQuietNAN(64'F(S0.f32)) then\nresult = S1.u32[1]\nelsif exponent(S0.f32) == 255 then\n// +-INF\nresult = S1.u32[sign(S0.f32) ? 2 : 9]\nelsif exponent(S0.f32) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f32) ? 3 : 8]\nelsif 64'F(abs(S0.f32)) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f32) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f32) ? 5 : 6]\nendif;\nEXEC.u64[laneId] = D0.u64[laneId] = result",
   VOPCOp.V_CMP_CLASS_F64_E32: "declare result : 1'U;\nif isSignalNAN(S0.f64) then\nresult = S1.u32[0]\nelsif isQuietNAN(S0.f64) then\nresult = S1.u32[1]\nelsif exponent(S0.f64) == 2047 then\n// +-INF\nresult = S1.u32[sign(S0.f64) ? 2 : 9]\nelsif exponent(S0.f64) > 0 then\n// +-normal value\nresult = S1.u32[sign(S0.f64) ? 3 : 8]\nelsif abs(S0.f64) > 0.0 then\n// +-denormal value\nresult = S1.u32[sign(S0.f64) ? 4 : 7]\nelse\n// +-0.0\nresult = S1.u32[sign(S0.f64) ? 5 : 6]\nendif;\nD0.u64[laneId] = result;\n// D0 = VCC in VOPC encoding.",

extra/assembly/amd/autogen/common.py

@@ -1,6 +1,10 @@
 # autogenerated from AMD ISA XML - do not edit
 from enum import Enum, auto
 
+class ReprEnum(Enum):
+  """Enum with clean repr that roundtrips with eval()."""
+  def __repr__(self): return f"{type(self).__name__}.{self.name}"
+
 class Fmt(Enum):
   FMT_ANY = auto()
   FMT_BUF = auto()

extra/assembly/amd/autogen/rdna3/enum.py

@@ -1,8 +1,7 @@
 # autogenerated from AMD ISA XML - do not edit
-from enum import Enum
-from extra.assembly.amd.autogen.common import Fmt, FMT_BITS, OpType  # noqa: F401
+from extra.assembly.amd.autogen.common import ReprEnum, Fmt, FMT_BITS, OpType  # noqa: F401
 
-class DSOp(Enum):
+class DSOp(ReprEnum):
   DS_ADD_U32 = 0
   DS_SUB_U32 = 1
   DS_RSUB_U32 = 2
@@ -130,10 +129,10 @@ class DSOp(Enum):
   DS_LOAD_B96 = 254
   DS_LOAD_B128 = 255
 
-class EXPOp(Enum):
+class EXPOp(ReprEnum):
   EXP = 0
 
-class FLATOp(Enum):
+class FLATOp(ReprEnum):
   FLAT_LOAD_U8 = 16
   FLAT_LOAD_I8 = 17
   FLAT_LOAD_U16 = 18
@@ -191,7 +190,7 @@ class FLATOp(Enum):
   FLAT_ATOMIC_MAX_F32 = 82
   FLAT_ATOMIC_ADD_F32 = 86
 
-class GLOBALOp(Enum):
+class GLOBALOp(ReprEnum):
   GLOBAL_LOAD_U8 = 16
   GLOBAL_LOAD_I8 = 17
   GLOBAL_LOAD_U16 = 18
@@ -254,11 +253,37 @@ class GLOBALOp(Enum):
   GLOBAL_ATOMIC_MAX_F32 = 82
   GLOBAL_ATOMIC_ADD_F32 = 86
 
-class LDSDIROp(Enum):
+class HWREG(ReprEnum):
+  HW_REG_MODE = 1
+  HW_REG_STATUS = 2
+  HW_REG_TRAPSTS = 3
+  HW_REG_GPR_ALLOC = 5
+  HW_REG_LDS_ALLOC = 6
+  HW_REG_IB_STS = 7
+  HW_REG_PC_LO = 8
+  HW_REG_PC_HI = 9
+  HW_REG_IB_DBG1 = 13
+  HW_REG_FLUSH_IB = 14
+  HW_REG_SH_MEM_BASES = 15
+  HW_REG_SHADER_TBA_LO = 16
+  HW_REG_SHADER_TBA_HI = 17
+  HW_REG_PERF_SNAPSHOT_PC_LO = 18
+  HW_REG_PERF_SNAPSHOT_PC_HI = 19
+  HW_REG_SHADER_FLAT_SCRATCH_LO = 20
+  HW_REG_SHADER_FLAT_SCRATCH_HI = 21
+  HW_REG_HW_ID1 = 23
+  HW_REG_HW_ID2 = 24
+  HW_REG_POPS_PACKER = 25
+  HW_REG_SCHED_MODE = 26
+  HW_REG_PERF_SNAPSHOT_DATA = 27
+  HW_REG_IB_STS2 = 28
+  HW_REG_SHADER_CYCLES = 29
+
+class LDSDIROp(ReprEnum):
   LDS_PARAM_LOAD = 0
   LDS_DIRECT_LOAD = 1
 
-class MIMGOp(Enum):
+class MIMGOp(ReprEnum):
   IMAGE_LOAD = 0
   IMAGE_LOAD_MIP = 1
   IMAGE_LOAD_PCK = 2
@@ -344,7 +369,17 @@ class MIMGOp(Enum):
   IMAGE_GATHER4_C_B_CL = 101
   IMAGE_GATHER4H = 144
 
-class MTBUFOp(Enum):
+class MSG(ReprEnum):
+  MSG_RTN_GET_DOORBELL = 128
+  MSG_RTN_GET_DDID = 129
+  MSG_RTN_GET_TMA = 130
+  MSG_RTN_GET_REALTIME = 131
+  MSG_RTN_SAVE_WAVE = 132
+  MSG_RTN_GET_TBA = 133
+  MSG_RTN_GET_TBA_TO_PC = 134
+  MSG_RTN_ILLEGAL_MSG = 255
+
+class MTBUFOp(ReprEnum):
   TBUFFER_LOAD_FORMAT_X = 0
   TBUFFER_LOAD_FORMAT_XY = 1
   TBUFFER_LOAD_FORMAT_XYZ = 2
@@ -362,7 +397,7 @@ class MTBUFOp(Enum):
   TBUFFER_STORE_D16_FORMAT_XYZ = 14
   TBUFFER_STORE_D16_FORMAT_XYZW = 15
 
-class MUBUFOp(Enum):
+class MUBUFOp(ReprEnum):
   BUFFER_LOAD_FORMAT_X = 0
   BUFFER_LOAD_FORMAT_XY = 1
   BUFFER_LOAD_FORMAT_XYZ = 2
@@ -443,7 +478,7 @@ class MUBUFOp(Enum):
   BUFFER_ATOMIC_MAX_F32 = 82
   BUFFER_ATOMIC_ADD_F32 = 86
 
-class SCRATCHOp(Enum):
+class SCRATCHOp(ReprEnum):
   SCRATCH_LOAD_U8 = 16
   SCRATCH_LOAD_I8 = 17
   SCRATCH_LOAD_U16 = 18
@@ -472,7 +507,7 @@ class SCRATCHOp(Enum):
   SCRATCH_LOAD_LDS_I16 = 48
   SCRATCH_LOAD_LDS_B32 = 49
 
-class SMEMOp(Enum):
+class SMEMOp(ReprEnum):
   S_LOAD_B32 = 0
   S_LOAD_B64 = 1
   S_LOAD_B128 = 2
@@ -488,7 +523,7 @@ class SMEMOp(Enum):
   S_ATC_PROBE = 34
   S_ATC_PROBE_BUFFER = 35
 
-class SOP1Op(Enum):
+class SOP1Op(ReprEnum):
   S_MOV_B32 = 0
   S_MOV_B64 = 1
   S_CMOV_B32 = 2
@@ -570,7 +605,7 @@ class SOP1Op(Enum):
   S_TRUNC_F16 = 109
   S_RNDNE_F16 = 110
 
-class SOP2Op(Enum):
+class SOP2Op(ReprEnum):
   S_ADD_U32 = 0
   S_SUB_U32 = 1
   S_ADD_I32 = 2
@@ -639,7 +674,7 @@ class SOP2Op(Enum):
   S_MUL_F16 = 77
   S_FMAC_F16 = 78
 
-class SOPCOp(Enum):
+class SOPCOp(ReprEnum):
   S_CMP_EQ_I32 = 0
   S_CMP_LG_I32 = 1
   S_CMP_GT_I32 = 2
@@ -687,7 +722,7 @@ class SOPCOp(Enum):
   S_CMP_NEQ_F16 = 93
   S_CMP_NLT_F16 = 94
 
-class SOPKOp(Enum):
+class SOPKOp(ReprEnum):
   S_MOVK_I32 = 0
   S_VERSION = 1
   S_CMOVK_I32 = 2
@@ -716,7 +751,7 @@ class SOPKOp(Enum):
   S_WAITCNT_EXPCNT = 26
   S_WAITCNT_LGKMCNT = 27
 
-class SOPPOp(Enum):
+class SOPPOp(ReprEnum):
   S_NOP = 0
   S_SETKILL = 1
   S_SETHALT = 2
@@ -757,7 +792,7 @@ class SOPPOp(Enum):
   S_ICACHE_INV = 60
   S_BARRIER = 61
 
-class VINTERPOp(Enum):
+class VINTERPOp(ReprEnum):
   V_INTERP_P10_F32 = 0
   V_INTERP_P2_F32 = 1
   V_INTERP_P10_F16_F32 = 2
@@ -765,7 +800,7 @@ class VINTERPOp(Enum):
   V_INTERP_P10_RTZ_F16_F32 = 4
   V_INTERP_P2_RTZ_F16_F32 = 5
 
-class VOP1Op(Enum):
+class VOP1Op(ReprEnum):
   V_NOP_E32 = 0
   V_MOV_B32_E32 = 1
   V_READFIRSTLANE_B32_E32 = 2
@@ -939,7 +974,7 @@ class VOP1Op(Enum):
   V_CVT_I32_I16 = V_CVT_I32_I16_E32
   V_CVT_U32_U16 = V_CVT_U32_U16_E32
 
-class VOP2Op(Enum):
+class VOP2Op(ReprEnum):
   V_CNDMASK_B32_E32 = 1
   V_DOT2ACC_F32_F16_E32 = 2
   V_ADD_F32_E32 = 3
@@ -1033,7 +1068,7 @@ class VOP2Op(Enum):
   V_LDEXP_F16 = V_LDEXP_F16_E32
   V_PK_FMAC_F16 = V_PK_FMAC_F16_E32
 
-class VOP3Op(Enum):
+class VOP3Op(ReprEnum):
   V_CMP_F_F16_E64 = 0
   V_CMP_LT_F16_E64 = 1
   V_CMP_EQ_F16_E64 = 2
@@ -1773,7 +1808,7 @@ class VOP3Op(Enum):
   V_CVT_I32_I16 = V_CVT_I32_I16_E64
   V_CVT_U32_U16 = V_CVT_U32_U16_E64
 
-class VOP3POp(Enum):
+class VOP3POp(ReprEnum):
   V_PK_MAD_I16 = 0
   V_PK_MUL_LO_U16 = 1
   V_PK_ADD_I16 = 2
@@ -1809,7 +1844,7 @@ class VOP3POp(Enum):
   V_WMMA_I32_16X16X16_IU8 = 68
   V_WMMA_I32_16X16X16_IU4 = 69
 
-class VOP3SDOp(Enum):
+class VOP3SDOp(ReprEnum):
   V_ADD_CO_CI_U32 = 288
   V_SUB_CO_CI_U32 = 289
   V_SUBREV_CO_CI_U32 = 290
@@ -1821,7 +1856,7 @@ class VOP3SDOp(Enum):
   V_SUB_CO_U32 = 769
   V_SUBREV_CO_U32 = 770
 
-class VOPCOp(Enum):
+class VOPCOp(ReprEnum):
   V_CMP_F_F16_E32 = 0
   V_CMP_LT_F16_E32 = 1
   V_CMP_EQ_F16_E32 = 2
@@ -2203,7 +2238,7 @@ class VOPCOp(Enum):
   V_CMPX_CLASS_F32 = V_CMPX_CLASS_F32_E32
   V_CMPX_CLASS_F64 = V_CMPX_CLASS_F64_E32
 
-class VOPDOp(Enum):
+class VOPDOp(ReprEnum):
   V_DUAL_FMAC_F32 = 0
   V_DUAL_FMAAK_F32 = 1
   V_DUAL_FMAMK_F32 = 2

extra/assembly/amd/autogen/rdna4/enum.py

@@ -1,8 +1,7 @@
 # autogenerated from AMD ISA XML - do not edit
-from enum import Enum
-from extra.assembly.amd.autogen.common import Fmt, FMT_BITS, OpType  # noqa: F401
+from extra.assembly.amd.autogen.common import ReprEnum, Fmt, FMT_BITS, OpType  # noqa: F401
 
-class DSOp(Enum):
+class DSOp(ReprEnum):
   DS_ADD_U32 = 0
   DS_SUB_U32 = 1
   DS_RSUB_U32 = 2
@@ -127,7 +126,39 @@ class DSOp(Enum):
   DS_LOAD_B96 = 254
   DS_LOAD_B128 = 255
 
-class SMEMOp(Enum):
+class HWREG(ReprEnum):
+  HW_REG_WAVE_MODE = 1
+  HW_REG_WAVE_STATUS = 2
+  HW_REG_WAVE_STATE_PRIV = 4
+  HW_REG_WAVE_GPR_ALLOC = 5
+  HW_REG_WAVE_LDS_ALLOC = 6
+  HW_REG_PERF_SNAPSHOT_DATA = 10
+  HW_REG_PERF_SNAPSHOT_PC_LO = 11
+  HW_REG_PERF_SNAPSHOT_PC_HI = 12
+  HW_REG_PERF_SNAPSHOT_DATA1 = 15
+  HW_REG_PERF_SNAPSHOT_DATA2 = 16
+  HW_REG_WAVE_EXCP_FLAG_PRIV = 17
+  HW_REG_WAVE_EXCP_FLAG_USER = 18
+  HW_REG_WAVE_TRAP_CTRL = 19
+  HW_REG_WAVE_SCRATCH_BASE_LO = 20
+  HW_REG_WAVE_SCRATCH_BASE_HI = 21
+  HW_REG_WAVE_HW_ID1 = 23
+  HW_REG_WAVE_HW_ID2 = 24
+  HW_REG_SHADER_CYCLES_LO = 29
+  HW_REG_SHADER_CYCLES_HI = 30
+
+class MSG(ReprEnum):
+  MSG_RTN_GET_DOORBELL = 128
+  MSG_RTN_GET_DDID = 129
+  MSG_RTN_GET_TMA = 130
+  MSG_RTN_GET_REALTIME = 131
+  MSG_RTN_SAVE_WAVE = 132
+  MSG_RTN_GET_TBA = 133
+  MSG_RTN_GET_TBA_TO_PC = 134
+  MSG_RTN_GET_SE_HW_ID = 135
+  MSG_RTN_ILLEGAL_MSG = 255
+
+class SMEMOp(ReprEnum):
   S_LOAD_B32 = 0
   S_LOAD_B64 = 1
   S_LOAD_B128 = 2
@@ -157,7 +188,7 @@ class SMEMOp(Enum):
   S_BUFFER_PREFETCH_DATA = 39
   S_PREFETCH_DATA_PC_REL = 40
 
-class SOP1Op(Enum):
+class SOP1Op(ReprEnum):
   S_MOV_B32 = 0
   S_MOV_B64 = 1
   S_CMOV_B32 = 2
@@ -246,7 +277,7 @@ class SOP1Op(Enum):
   S_TRUNC_F16 = 109
   S_RNDNE_F16 = 110
 
-class SOP2Op(Enum):
+class SOP2Op(ReprEnum):
   S_ADD_CO_U32 = 0
   S_SUB_CO_U32 = 1
   S_ADD_CO_I32 = 2
@@ -322,7 +353,7 @@ class SOP2Op(Enum):
   S_SUB_NC_U64 = 84
   S_MUL_U64 = 85
 
-class SOPCOp(Enum):
+class SOPCOp(ReprEnum):
   S_CMP_EQ_I32 = 0
   S_CMP_LG_I32 = 1
   S_CMP_GT_I32 = 2
@@ -370,7 +401,7 @@ class SOPCOp(Enum):
   S_CMP_NEQ_F16 = 93
   S_CMP_NLT_F16 = 94
 
-class SOPKOp(Enum):
+class SOPKOp(ReprEnum):
   S_MOVK_I32 = 0
   S_VERSION = 1
   S_CMOVK_I32 = 2
@@ -381,7 +412,7 @@ class SOPKOp(Enum):
   S_SETREG_IMM32_B32 = 19
   S_CALL_B64 = 20
 
-class SOPPOp(Enum):
+class SOPPOp(ReprEnum):
   S_NOP = 0
   S_SETKILL = 1
   S_SETHALT = 2
@@ -426,7 +457,7 @@ class SOPPOp(Enum):
   S_WAIT_LOADCNT_DSCNT = 72
   S_WAIT_STORECNT_DSCNT = 73
 
-class VBUFFEROp(Enum):
+class VBUFFEROp(ReprEnum):
   BUFFER_LOAD_FORMAT_X = 0
   BUFFER_LOAD_FORMAT_XY = 1
   BUFFER_LOAD_FORMAT_XYZ = 2
@@ -517,14 +548,14 @@ class VBUFFEROp(Enum):
   TBUFFER_STORE_D16_FORMAT_XYZ = 142
   TBUFFER_STORE_D16_FORMAT_XYZW = 143
 
-class VDSDIROp(Enum):
+class VDSDIROp(ReprEnum):
   DS_PARAM_LOAD = 0
   DS_DIRECT_LOAD = 1
 
-class VEXPORTOp(Enum):
+class VEXPORTOp(ReprEnum):
   EXPORT = 0
 
-class VFLATOp(Enum):
+class VFLATOp(ReprEnum):
   FLAT_LOAD_U8 = 16
   FLAT_LOAD_I8 = 17
   FLAT_LOAD_U16 = 18
@@ -583,7 +614,7 @@ class VFLATOp(Enum):
   FLAT_ATOMIC_PK_ADD_F16 = 89
   FLAT_ATOMIC_PK_ADD_BF16 = 90
 
-class VGLOBALOp(Enum):
+class VGLOBALOp(ReprEnum):
   GLOBAL_LOAD_U8 = 16
   GLOBAL_LOAD_I8 = 17
   GLOBAL_LOAD_U16 = 18
@@ -650,7 +681,7 @@ class VGLOBALOp(Enum):
   GLOBAL_ATOMIC_PK_ADD_BF16 = 90
   GLOBAL_ATOMIC_ORDERED_ADD_B64 = 115
 
-class VIMAGEOp(Enum):
+class VIMAGEOp(ReprEnum):
   IMAGE_LOAD = 0
   IMAGE_LOAD_MIP = 1
   IMAGE_LOAD_PCK = 2
@@ -685,7 +716,7 @@ class VIMAGEOp(Enum):
   IMAGE_ATOMIC_PK_ADD_F16 = 134
   IMAGE_ATOMIC_PK_ADD_BF16 = 135
 
-class VINTERPOp(Enum):
+class VINTERPOp(ReprEnum):
   V_INTERP_P10_F32 = 0
   V_INTERP_P2_F32 = 1
   V_INTERP_P10_F16_F32 = 2
@@ -693,7 +724,7 @@ class VINTERPOp(Enum):
   V_INTERP_P10_RTZ_F16_F32 = 4
   V_INTERP_P2_RTZ_F16_F32 = 5
 
-class VOP1Op(Enum):
+class VOP1Op(ReprEnum):
   V_NOP_E32 = 0
   V_MOV_B32_E32 = 1
   V_READFIRSTLANE_B32_E32 = 2
@@ -875,7 +906,7 @@ class VOP1Op(Enum):
   V_CVT_PK_F32_FP8 = V_CVT_PK_F32_FP8_E32
   V_CVT_PK_F32_BF8 = V_CVT_PK_F32_BF8_E32
 
-class VOP2Op(Enum):
+class VOP2Op(ReprEnum):
   V_CNDMASK_B32_E32 = 1
   V_ADD_F64_E32 = 2
   V_ADD_F32_E32 = 3
@@ -975,7 +1006,7 @@ class VOP2Op(Enum):
   V_LDEXP_F16 = V_LDEXP_F16_E32
   V_PK_FMAC_F16 = V_PK_FMAC_F16_E32
 
-class VOP3Op(Enum):
+class VOP3Op(ReprEnum):
   V_CMP_LT_F16_E64 = 1
   V_CMP_EQ_F16_E64 = 2
   V_CMP_LE_F16_E64 = 3
@@ -1700,7 +1731,7 @@ class VOP3Op(Enum):
   V_CVT_PK_F32_FP8 = V_CVT_PK_F32_FP8_E64
   V_CVT_PK_F32_BF8 = V_CVT_PK_F32_BF8_E64
 
-class VOP3POp(Enum):
+class VOP3POp(ReprEnum):
   V_PK_MAD_I16 = 0
   V_PK_MUL_LO_U16 = 1
   V_PK_ADD_I16 = 2
@@ -1758,7 +1789,7 @@ class VOP3POp(Enum):
   V_SWMMAC_F32_16X16X32_BF8_FP8 = 89
   V_SWMMAC_F32_16X16X32_BF8_BF8 = 90
 
-class VOP3SDOp(Enum):
+class VOP3SDOp(ReprEnum):
   V_ADD_CO_CI_U32 = 288
   V_SUB_CO_CI_U32 = 289
   V_SUBREV_CO_CI_U32 = 290
@@ -1770,7 +1801,7 @@ class VOP3SDOp(Enum):
   V_SUB_CO_U32 = 769
   V_SUBREV_CO_U32 = 770
 
-class VOPCOp(Enum):
+class VOPCOp(ReprEnum):
   V_CMP_LT_F16_E32 = 1
   V_CMP_EQ_F16_E32 = 2
   V_CMP_LE_F16_E32 = 3
@@ -2096,7 +2127,7 @@ class VOPCOp(Enum):
   V_CMPX_CLASS_F32 = V_CMPX_CLASS_F32_E32
   V_CMPX_CLASS_F64 = V_CMPX_CLASS_F64_E32
 
-class VOPDOp(Enum):
+class VOPDOp(ReprEnum):
   V_DUAL_FMAC_F32 = 0
   V_DUAL_FMAAK_F32 = 1
   V_DUAL_FMAMK_F32 = 2
@@ -2115,7 +2146,7 @@ class VOPDOp(Enum):
   V_DUAL_LSHLREV_B32 = 17
   V_DUAL_AND_B32 = 18
 
-class VSAMPLEOp(Enum):
+class VSAMPLEOp(ReprEnum):
   IMAGE_MSAA_LOAD = 24
   IMAGE_SAMPLE = 27
   IMAGE_SAMPLE_D = 28
@@ -2175,7 +2206,7 @@ class VSAMPLEOp(Enum):
   IMAGE_GATHER4_C_B_CL = 101
   IMAGE_GATHER4H = 144
 
-class VSCRATCHOp(Enum):
+class VSCRATCHOp(ReprEnum):
   SCRATCH_LOAD_U8 = 16
   SCRATCH_LOAD_I8 = 17
   SCRATCH_LOAD_U16 = 18

extra/assembly/amd/decode.py

@@ -1,83 +0,0 @@
-# Instruction format detection and decoding
-from __future__ import annotations
-from extra.assembly.amd.dsl import Inst, FixedBitField
-from extra.assembly.amd.autogen.rdna3.ins import (VOP1, VOP1_SDST, VOP2, VOP3, VOP3_SDST, VOP3SD, VOP3P, VOPC, VOPD, VINTERP,
-  SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, DS, FLAT)
-from extra.assembly.amd.autogen.rdna4.ins import (VOP1 as R4_VOP1, VOP1_SDST as R4_VOP1_SDST, VOP2 as R4_VOP2,
-  VOP3 as R4_VOP3, VOP3_SDST as R4_VOP3_SDST, VOP3SD as R4_VOP3SD, VOP3P as R4_VOP3P,
-  VOPC as R4_VOPC, VOPD as R4_VOPD, VINTERP as R4_VINTERP, SOP1 as R4_SOP1, SOP2 as R4_SOP2, SOPC as R4_SOPC, SOPK as R4_SOPK, SOPP as R4_SOPP,
-  SMEM as R4_SMEM, DS as R4_DS)
-from extra.assembly.amd.autogen.cdna.ins import (VOP1 as C_VOP1, VOP2 as C_VOP2, VOPC as C_VOPC, VOP3A, VOP3B, VOP3P as C_VOP3P,
-  SOP1 as C_SOP1, SOP2 as C_SOP2, SOPC as C_SOPC, SOPK as C_SOPK, SOPP as C_SOPP, SMEM as C_SMEM, DS as C_DS, FLAT as C_FLAT)
-
-def _matches_encoding(word: int, cls: type[Inst]) -> bool:
-  """Check if word matches the encoding pattern of an instruction class."""
-  enc = next(((n, f) for n, f in cls._fields if isinstance(f, FixedBitField) and n == 'encoding'), None)
-  if enc is None: return False
-  bf = enc[1]
-  return ((word >> bf.lo) & bf.mask) == bf.default
-
-# Order matters: more specific encodings first, VOP2 last (it's a catch-all for bit31=0)
-_RDNA_FORMATS_64 = [VOPD, VOP3P, VINTERP, VOP3, DS, FLAT, SMEM]
-_RDNA_FORMATS_32 = [SOP1, SOPC, SOPP, SOPK, VOPC, VOP1, SOP2, VOP2]  # SOP2/VOP2 are catch-alls
-_CDNA_FORMATS_64 = [C_VOP3P, VOP3A, C_DS, C_FLAT, C_SMEM]
-_CDNA_FORMATS_32 = [C_SOP1, C_SOPC, C_SOPP, C_SOPK, C_VOPC, C_VOP1, C_SOP2, C_VOP2]
-_CDNA_VOP3B_OPS = {281, 282, 283, 284, 285, 286, 480, 481, 488, 489}  # VOP3B opcodes
-_RDNA4_FORMATS_64 = [R4_VOPD, R4_VOP3P, R4_VINTERP, R4_VOP3, R4_DS, R4_SMEM]
-_RDNA4_FORMATS_32 = [R4_SOP1, R4_SOPC, R4_SOPP, R4_SOPK, R4_VOPC, R4_VOP1, R4_SOP2, R4_VOP2]
-_RDNA4_VOP3SD_OPS = {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}
-_RDNA3_VOP3SD_OPS = {288, 289, 290, 764, 765, 766, 767, 768, 769, 770}
-# Instructions with SGPR destination (READLANE, READFIRSTLANE, and VOP3-encoded VOPC)
-_VOP1_SDST_OPS = {2}  # V_READFIRSTLANE_B32_E32
-_VOP3_SDST_OPS = {386, 864}  # V_READFIRSTLANE_B32_E64, V_READLANE_B32 (V_WRITELANE_B32=865 writes to VGPR)
-# VOP3-encoded VOPC instructions (opcodes < 256) also have SGPR destination
-
-def detect_format(data: bytes, arch: str = "rdna3") -> type[Inst]:
-  """Detect instruction format from machine code bytes."""
-  assert len(data) >= 4, f"need at least 4 bytes, got {len(data)}"
-  word = int.from_bytes(data[:4], 'little')
-  if arch == "cdna":
-    # SDWA (0xf9) and DPP (0xfa) use special src0 encodings - not supported
-    if (word & 0x1ff) in (0xf9, 0xfa): raise ValueError(f"CDNA SDWA/DPP not supported word={word:#010x}")
-    if (word >> 30) == 0b11:
-      for cls in _CDNA_FORMATS_64:
-        if _matches_encoding(word, cls):
-          return VOP3B if cls is VOP3A and ((word >> 16) & 0x3ff) in _CDNA_VOP3B_OPS else cls
-      raise ValueError(f"unknown CDNA 64-bit format word={word:#010x}")
-    for cls in _CDNA_FORMATS_32:
-      if _matches_encoding(word, cls): return cls
-    raise ValueError(f"unknown CDNA 32-bit format word={word:#010x}")
-  if arch == "rdna4":
-    if (word >> 30) == 0b11:
-      for cls in _RDNA4_FORMATS_64:
-        if _matches_encoding(word, cls):
-          if cls is R4_VOP3:
-            opcode = (word >> 16) & 0x3ff
-            if opcode in _RDNA4_VOP3SD_OPS: return R4_VOP3SD
-            if opcode in _VOP3_SDST_OPS or opcode < 256: return R4_VOP3_SDST  # VOP3-encoded VOPC (op < 256) writes to SGPR
-          return cls
-      raise ValueError(f"unknown RDNA4 64-bit format word={word:#010x}")
-    for cls in _RDNA4_FORMATS_32:
-      if _matches_encoding(word, cls):
-        if cls is R4_VOP1 and ((word >> 9) & 0xff) in _VOP1_SDST_OPS: return R4_VOP1_SDST
-        return cls
-    raise ValueError(f"unknown RDNA4 32-bit format word={word:#010x}")
-  # RDNA3 (default)
-  if (word >> 30) == 0b11:
-    for cls in _RDNA_FORMATS_64:
-      if _matches_encoding(word, cls):
-        if cls is VOP3:
-          opcode = (word >> 16) & 0x3ff
-          if opcode in _RDNA3_VOP3SD_OPS: return VOP3SD
-          if opcode in _VOP3_SDST_OPS or opcode < 256: return VOP3_SDST  # VOP3-encoded VOPC (op < 256) writes to SGPR
-        return cls
-    raise ValueError(f"unknown 64-bit format word={word:#010x}")
-  for cls in _RDNA_FORMATS_32:
-    if _matches_encoding(word, cls):
-      if cls is VOP1 and ((word >> 9) & 0xff) in _VOP1_SDST_OPS: return VOP1_SDST
-      return cls
-  raise ValueError(f"unknown 32-bit format word={word:#010x}")
-
-def decode_inst(data: bytes, arch: str = "rdna3") -> Inst:
-  """Decode machine code bytes into an instruction."""
-  return detect_format(data, arch).from_bytes(data)

extra/assembly/amd/disasm.py

@@ -1,14 +1,16 @@
 # RDNA3/RDNA4/CDNA disassembler
 from __future__ import annotations
 import re, struct
+from typing import Callable
 from extra.assembly.amd.dsl import Inst, Reg
 
 # Special register mappings for disassembly
 SPECIAL_GPRS = {106: 'vcc_lo', 107: 'vcc_hi', 124: 'null', 125: 'm0', 126: 'exec_lo', 127: 'exec_hi',
                 128: '0', 240: '0.5', 241: '-0.5', 242: '1.0', 243: '-1.0', 244: '2.0', 245: '-2.0', 246: '4.0', 247: '-4.0', 248: '0x3e22f983', 253: 'scc'}
-SPECIAL_GPRS_CDNA = {106: 'vcc_lo', 107: 'vcc_hi', 124: 'null', 125: 'm0', 126: 'exec_lo', 127: 'exec_hi',
+SPECIAL_GPRS_CDNA = {106: 'vcc_lo', 107: 'vcc_hi', 124: 'm0', 126: 'exec_lo', 127: 'exec_hi',
                      128: '0', 240: '0.5', 241: '-0.5', 242: '1.0', 243: '-1.0', 244: '2.0', 245: '-2.0', 246: '4.0', 247: '-4.0', 248: '0x3e22f983', 253: 'scc',
-                     102: 'flat_scratch_lo', 103: 'flat_scratch_hi', 104: 'xnack_mask_lo', 105: 'xnack_mask_hi'}
+                     102: 'flat_scratch_lo', 103: 'flat_scratch_hi', 104: 'xnack_mask_lo', 105: 'xnack_mask_hi',
+                     251: 'src_vccz', 252: 'src_execz'}
 SPECIAL_PAIRS = {106: 'vcc', 126: 'exec'}
 SPECIAL_PAIRS_CDNA = {106: 'vcc', 126: 'exec', 102: 'flat_scratch', 104: 'xnack_mask'}
 
@@ -39,8 +41,9 @@ def _vi(v) -> int:
 
 _FLOAT_DEC = {240: 0.5, 241: -0.5, 242: 1.0, 243: -1.0, 244: 2.0, 245: -2.0, 246: 4.0, 247: -4.0}
 
-def _lit(inst, v, neg=0) -> str:
+def _lit(inst, v, neg=0, cdna=None) -> str:
   """Format literal/inline constant value."""
+  if cdna is None: cdna = _is_cdna(inst)
   v = _unwrap(v)
   if v == 255:
     lit = inst._literal
@@ -49,9 +52,9 @@ def _lit(inst, v, neg=0) -> str:
   elif v in _FLOAT_DEC: s = str(_FLOAT_DEC[v])
   elif 128 <= v <= 192: s = str(v - 128)
   elif 193 <= v <= 208: s = str(-(v - 192))
-  elif v < 128: s = decode_src(v)
+  elif v < 128: s = decode_src(v, cdna)
   elif v >= 256: s = f"v{v - 256}"
-  else: s = decode_src(v)
+  else: s = decode_src(v, cdna)
   return f"-{s}" if neg else s
 
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -70,47 +73,24 @@ def _num_srcs(inst) -> int:
   if any(x in n for x in ('_ADD3', '_LSHL_ADD', '_ADD_LSHL', '_LSHL_OR', '_AND_OR', 'OR3_B32', 'AND_OR_B32', 'ALIGNBIT', 'ALIGNBYTE', 'V_PERM_', 'XOR3', 'XAD', 'MULLIT', 'MINMAX', 'MAXMIN', 'MINIMUMMAXIMUM', 'MAXIMUMMINIMUM', 'MINIMUM3', 'MAXIMUM3', 'MIN3', 'MAX3', 'DOT2', 'CVT_PK_U8_F32', 'DOT4', 'DOT8', 'WMMA', 'SWMMAC')): return 3
   return 2
 
-# SWMMAC register counts: (dst, src0, src1, src2)
-def _swmmac_regs(name: str) -> tuple[int, int, int, int]:
-  """Return (dst, src0, src1, src2) register counts for SWMMAC instructions."""
-  if 'f16_16x16x32' in name or 'bf16_16x16x32' in name: return (4, 4, 8, 1)
-  if 'f32_16x16x32_f16' in name or 'f32_16x16x32_bf16' in name: return (8, 4, 8, 1)
-  if 'i32_16x16x32_iu4' in name: return (8, 1, 2, 1)
-  if 'i32_16x16x64_iu4' in name: return (8, 2, 4, 1)
-  if 'i32_16x16x32_iu8' in name or 'f32_16x16x32_fp8' in name or 'f32_16x16x32_bf8' in name: return (8, 2, 4, 1)
-  return (8, 8, 8, 8)
-
 # ═══════════════════════════════════════════════════════════════════════════════
 # IMPORTS
 # ═══════════════════════════════════════════════════════════════════════════════
 
-from extra.assembly.amd.autogen.rdna3.ins import (VOP1, VOP1_SDST, VOP2, VOP3, VOP3_SDST, VOP3SD, VOP3P, VOPC, VOPD, VINTERP, SOP1, SOP2, SOPC, SOPK, SOPP, SMEM, DS, FLAT, GLOBAL, SCRATCH,
-  VOP1Op, VOP2Op, VOP3Op, VOP3SDOp, VOPDOp, SOP1Op, SOPKOp, SOPPOp, SMEMOp, DSOp)
-from extra.assembly.amd.autogen.rdna4.ins import (VOP1 as R4_VOP1, VOP1_SDST as R4_VOP1_SDST, VOP2 as R4_VOP2, VOP3 as R4_VOP3, VOP3_SDST as R4_VOP3_SDST, VOP3SD as R4_VOP3SD, VOP3P as R4_VOP3P,
-  VOPC as R4_VOPC, VOPD as R4_VOPD, VINTERP as R4_VINTERP, SOP1 as R4_SOP1, SOP2 as R4_SOP2, SOPC as R4_SOPC, SOPK as R4_SOPK, SOPP as R4_SOPP,
-  SMEM as R4_SMEM, DS as R4_DS, VOPDOp as R4_VOPDOp)
-from extra.assembly.amd.autogen.cdna.ins import FLAT as C_FLAT
+from extra.assembly.amd.autogen.rdna3.ins import (VOP1, VOP1_SDST, VOP1_SDST_LIT, VOP1_LIT, VOP2, VOP2_LIT, VOP3, VOP3_SDST, VOP3_SDST_LIT,
+  VOP3_LIT, VOP3SD, VOP3SD_LIT, VOP3P, VOP3P_LIT, VOPC, VOPC_LIT, VOPD, VOPD_LIT, VINTERP, SOP1, SOP1_LIT, SOP2, SOP2_LIT, SOPC, SOPC_LIT,
+  SOPK, SOPK_LIT, SOPP, SMEM, DS, FLAT, GLOBAL, SCRATCH, VOP2Op, VOPDOp, SOPPOp, HWREG, MSG)
+from extra.assembly.amd.autogen.rdna4.ins import (VOP1 as R4_VOP1, VOP1_SDST as R4_VOP1_SDST, VOP1_SDST_LIT as R4_VOP1_SDST_LIT, VOP1_LIT as R4_VOP1_LIT,
+  VOP2 as R4_VOP2, VOP2_LIT as R4_VOP2_LIT, VOP3 as R4_VOP3, VOP3_SDST as R4_VOP3_SDST, VOP3_SDST_LIT as R4_VOP3_SDST_LIT, VOP3_LIT as R4_VOP3_LIT,
+  VOP3SD as R4_VOP3SD, VOP3SD_LIT as R4_VOP3SD_LIT, VOP3P as R4_VOP3P, VOP3P_LIT as R4_VOP3P_LIT, VOPC as R4_VOPC, VOPC_LIT as R4_VOPC_LIT,
+  VOPD as R4_VOPD, VOPD_LIT as R4_VOPD_LIT, VINTERP as R4_VINTERP, SOP1 as R4_SOP1, SOP1_LIT as R4_SOP1_LIT, SOP2 as R4_SOP2, SOP2_LIT as R4_SOP2_LIT,
+  SOPC as R4_SOPC, SOPC_LIT as R4_SOPC_LIT, SOPK as R4_SOPK, SOPK_LIT as R4_SOPK_LIT, SOPP as R4_SOPP, SMEM as R4_SMEM, DS as R4_DS,
+  VOPDOp as R4_VOPDOp, HWREG as HWREG_RDNA4, VFLAT as R4_FLAT, VGLOBAL as R4_GLOBAL, VSCRATCH as R4_SCRATCH)
+from extra.assembly.amd.autogen.cdna.ins import FLAT as C_FLAT, HWREG as HWREG_CDNA
 
 def _is_cdna(inst: Inst) -> bool: return 'cdna' in inst.__class__.__module__
+def _is_r4(inst: Inst) -> bool: return 'rdna4' in inst.__class__.__module__
 
-# ═══════════════════════════════════════════════════════════════════════════════
-# CONSTANTS
-# ═══════════════════════════════════════════════════════════════════════════════
-
-HWREG = {1: 'HW_REG_MODE', 2: 'HW_REG_STATUS', 3: 'HW_REG_TRAPSTS', 4: 'HW_REG_HW_ID', 5: 'HW_REG_GPR_ALLOC',
-         6: 'HW_REG_LDS_ALLOC', 7: 'HW_REG_IB_STS', 15: 'HW_REG_SH_MEM_BASES', 18: 'HW_REG_PERF_SNAPSHOT_PC_LO',
-         19: 'HW_REG_PERF_SNAPSHOT_PC_HI', 20: 'HW_REG_FLAT_SCR_LO', 21: 'HW_REG_FLAT_SCR_HI', 22: 'HW_REG_XNACK_MASK',
-         23: 'HW_REG_HW_ID1', 24: 'HW_REG_HW_ID2', 25: 'HW_REG_POPS_PACKER', 28: 'HW_REG_IB_STS2'}
-HWREG_RDNA4 = {1: 'HW_REG_MODE', 2: 'HW_REG_STATUS', 4: 'HW_REG_STATE_PRIV', 5: 'HW_REG_GPR_ALLOC',
-               6: 'HW_REG_LDS_ALLOC', 7: 'HW_REG_IB_STS', 10: 'HW_REG_PERF_SNAPSHOT_DATA', 11: 'HW_REG_PERF_SNAPSHOT_PC_LO',
-               12: 'HW_REG_PERF_SNAPSHOT_PC_HI', 15: 'HW_REG_PERF_SNAPSHOT_DATA1', 16: 'HW_REG_PERF_SNAPSHOT_DATA2',
-               17: 'HW_REG_EXCP_FLAG_PRIV', 18: 'HW_REG_EXCP_FLAG_USER', 19: 'HW_REG_TRAP_CTRL',
-               20: 'HW_REG_SCRATCH_BASE_LO', 21: 'HW_REG_SCRATCH_BASE_HI', 23: 'HW_REG_HW_ID1',
-               24: 'HW_REG_HW_ID2', 26: 'HW_REG_SCHED_MODE', 29: 'HW_REG_SHADER_CYCLES_LO',
-               30: 'HW_REG_SHADER_CYCLES_HI', 31: 'HW_REG_DVGPR_ALLOC_LO', 32: 'HW_REG_DVGPR_ALLOC_HI'}
-MSG = {128: 'MSG_RTN_GET_DOORBELL', 129: 'MSG_RTN_GET_DDID', 130: 'MSG_RTN_GET_TMA',
-       131: 'MSG_RTN_GET_REALTIME', 132: 'MSG_RTN_SAVE_WAVE', 133: 'MSG_RTN_GET_TBA',
-       134: 'MSG_RTN_GET_TBA_TO_PC', 135: 'MSG_RTN_GET_SE_AID_ID'}
 # CDNA opcode name aliases for disasm (new name -> old name expected by tests)
 _CDNA_DISASM_ALIASES = {'v_fmac_f64': 'v_mul_legacy_f32', 'v_dot2c_f32_bf16': 'v_mac_f32', 'v_fmamk_f32': 'v_madmk_f32', 'v_fmaak_f32': 'v_madak_f32'}
 
@@ -122,8 +102,7 @@ def _reg(p: str, b: int, n: int = 1) -> str: return f"{p}{_unwrap(b)}" if n == 1
 def _sreg(b: int, n: int = 1) -> str: return _reg("s", _unwrap(b), n)
 def _vreg(b: int, n: int = 1) -> str: b = _unwrap(b); return _reg("v", b - 256 if b >= 256 else b, n)
 def _areg(b: int, n: int = 1) -> str: b = _unwrap(b); return _reg("a", b - 256 if b >= 256 else b, n)  # accumulator registers for GFX90a
-def _ttmp(b, n: int = 1) -> str: b = _unwrap(b); return _reg("ttmp", b - 108, n) if 108 <= b <= 123 else None
-def _sreg_or_ttmp(b, n: int = 1) -> str: return _ttmp(b, n) or _sreg(b, n)
+def _ttmp(b, n: int = 1) -> str | None: b = _unwrap(b); return _reg("ttmp", b - 108, n) if 108 <= b <= 123 else None
 
 def _fmt_sdst(v, n: int = 1, cdna: bool = False) -> str:
   v = _unwrap(v)
@@ -171,7 +150,8 @@ def _opsel_str(opsel: int, n: int, need: bool, is16_d: bool) -> str:
   if not need: return ""
   dst_hi = (opsel >> 3) & 1
   if n == 1: return f" op_sel:[{opsel & 1},{dst_hi}]"
-  if n == 2: return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{dst_hi}]"
+  # Use 4-element format if bit 2 is set (src2 selection used) or if 3+ sources
+  if n == 2 and not ((opsel >> 2) & 1): return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{dst_hi}]"
   return f" op_sel:[{opsel & 1},{(opsel >> 1) & 1},{(opsel >> 2) & 1},{dst_hi}]"
 
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -188,8 +168,6 @@ def _disasm_vop1(inst: VOP1) -> str:
     return f"{name} {_fmt_sdst(vdst_off, 1, cdna)}, {src}"
   bits = inst.canonical_op_bits
   is16_dst, is16_src = not cdna and bits['d'] == 16, not cdna and bits['s0'] == 16
-  # v_cvt_pk_f32_fp8/bf8: pcode has None dst type but outputs 2 VGPRs
-  if 'cvt_pk_f32_fp8' in name or 'cvt_pk_f32_bf8' in name: is16_src = True
   # Format dst
   if is16_dst: dst = _fmt_v16(inst.vdst)
   else: dst = inst.vdst.fmt()
@@ -207,7 +185,7 @@ def _disasm_vop2(inst: VOP2) -> str:
   name, cdna = inst.op_name.lower(), _is_cdna(inst)
   if cdna: name = _CDNA_DISASM_ALIASES.get(name, name)  # apply CDNA aliases
   suf = "" if cdna or name.endswith('_e32') or (not cdna and inst.op == VOP2Op.V_DOT2ACC_F32_F16_E32) else "_e32"
-  lit = getattr(inst, '_literal', None)
+  lit = inst._literal
   is16 = not cdna and inst.canonical_op_bits['d'] == 16
   # fmaak/madak: dst = src0 * vsrc1 + K, fmamk/madmk: dst = src0 * K + vsrc1
   if 'fmaak' in name or 'madak' in name or (not cdna and inst.op in (VOP2Op.V_FMAAK_F32_E32, VOP2Op.V_FMAAK_F16_E32)):
@@ -220,9 +198,10 @@ def _disasm_vop2(inst: VOP2) -> str:
     return f"{name}{suf} {inst.vdst.fmt()}, {_lit(inst, inst.src0)}, 0x{lit:x}, {inst.vsrc1.fmt()}"
   if is16: return f"{name}{suf} {_fmt_v16(inst.vdst)}, {_src16(inst, inst.src0)}, {_fmt_v16(inst.vsrc1)}"
   vcc = "vcc" if cdna else "vcc_lo"
-  if cdna and name in _VOP2_CARRY_OUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}"
-  if cdna and name in _VOP2_CARRY_INOUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
-  if not cdna and name in _VOP2_CARRY_INOUT_RDNA: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
+  basename = name.replace('_e32', '')
+  if cdna and basename in _VOP2_CARRY_OUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}"
+  if cdna and basename in _VOP2_CARRY_INOUT: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
+  if not cdna and basename in _VOP2_CARRY_INOUT_RDNA: return f"{name}{suf} {inst.vdst.fmt()}, {vcc}, {_lit(inst, inst.src0)}, {inst.vsrc1.fmt()}, {vcc}"
   sn0 = inst.canonical_op_regs.get('s0', 1)
   if inst.vdst.sz > 1 or sn0 > 1 or inst.vsrc1.sz > 1:
     src0 = _lit(inst, inst.src0) if inst.src0.offset == 255 else _fmt_src(inst.src0, sn0, cdna)
@@ -245,12 +224,10 @@ def _disasm_vopc(inst: VOPC) -> str:
 
 NO_ARG_SOPP = {SOPPOp.S_BARRIER, SOPPOp.S_WAKEUP, SOPPOp.S_ICACHE_INV,
                SOPPOp.S_WAIT_IDLE, SOPPOp.S_ENDPGM_SAVED, SOPPOp.S_CODE_END, SOPPOp.S_ENDPGM_ORDERED_PS_DONE, SOPPOp.S_TTRACEDATA}
-_CDNA_NO_ARG_SOPP = {'s_endpgm', 's_barrier', 's_wakeup', 's_icache_inv', 's_ttracedata', 's_nop', 's_sethalt', 's_sleep',
-                     's_setprio', 's_trap', 's_incperflevel', 's_decperflevel', 's_sendmsg', 's_sendmsghalt'}
 
 def _disasm_sopp(inst: SOPP) -> str:
   name, cdna = inst.op_name.lower(), _is_cdna(inst)
-  is_rdna4 = 'rdna4' in inst.__class__.__module__
+  is_rdna4 = _is_r4(inst)
   # Ops that have no argument when simm16 == 0
   no_arg_zero = {'s_barrier', 's_wakeup', 's_icache_inv', 's_ttracedata', 's_wait_idle', 's_endpgm_saved',
                  's_endpgm_ordered_ps_done', 's_code_end'}
@@ -258,10 +235,15 @@ def _disasm_sopp(inst: SOPP) -> str:
   if name == 's_endpgm': return name if inst.simm16 == 0 else f"{name} {inst.simm16}"
   if cdna:
     if name == 's_waitcnt':
-      vm, lgkm, exp = inst.simm16 & 0xf, (inst.simm16 >> 8) & 0x3f, (inst.simm16 >> 4) & 0x7
-      p = [f"vmcnt({vm})" if vm != 0xf else "", f"expcnt({exp})" if exp != 7 else "", f"lgkmcnt({lgkm})" if lgkm != 0x3f else ""]
+      # GFX9 format: vmcnt[3:0]=bits[3:0], vmcnt[5:4]=bits[15:14], expcnt=bits[6:4], lgkmcnt=bits[11:8] (4 bits, max 15)
+      vm_lo, exp, lgkm, vm_hi = inst.simm16 & 0xf, (inst.simm16 >> 4) & 0x7, (inst.simm16 >> 8) & 0xf, (inst.simm16 >> 14) & 0x3
+      vm = vm_lo | (vm_hi << 4)
+      p = [f"vmcnt({vm})" if vm != 0x3f else "", f"expcnt({exp})" if exp != 7 else "", f"lgkmcnt({lgkm})" if lgkm != 0xf else ""]
       return f"s_waitcnt {' '.join(x for x in p if x) or '0'}"
     if name.startswith(('s_cbranch', 's_branch')): return f"{name} {inst.simm16}"
+    if name == 's_set_gpr_idx_mode':
+      flags = [n for i, n in enumerate(['SRC0', 'SRC1', 'SRC2', 'DST']) if inst.simm16 & (1 << i)]
+      return f"{name} gpr_idx({','.join(flags)})"
     return f"{name} 0x{inst.simm16:x}" if inst.simm16 else name
   # RDNA (use name-based checks instead of enum-based for cross-arch compatibility)
   if name == 's_waitcnt':
@@ -277,14 +259,14 @@ def _disasm_sopp(inst: SOPP) -> str:
     dep = lambda v: deps[v-1] if 0 < v <= len(deps) else str(v)
     p = [f"instid0({dep(id0)})" if id0 else "", f"instskip({skips[skip]})" if skip else "", f"instid1({dep(id1)})" if id1 else ""]
     return f"s_delay_alu {' | '.join(x for x in p if x) or '0'}"
-  if name.startswith(('s_cbranch', 's_branch')): return f"{name} 0x{inst.simm16:x}"
+  if name.startswith(('s_cbranch', 's_branch')): return f"{name} {inst.simm16}"
   return f"{name} 0x{inst.simm16:x}"
 
 def _disasm_smem(inst: SMEM) -> str:
   name, cdna = inst.op_name.lower(), _is_cdna(inst)
-  if name in ('s_gl1_inv', 's_dcache_inv'): return name
-  soe, imm = getattr(inst, 'soe', 0), getattr(inst, 'imm', 1)
-  is_rdna4 = 'rdna4' in inst.__class__.__module__
+  if name in ('s_gl1_inv', 's_dcache_inv', 's_dcache_inv_vol', 's_dcache_wb', 's_dcache_wb_vol', 's_icache_inv'): return name
+  soe, imm = getattr(inst, 'soe', 0) or getattr(inst, 'soffset_en', 0), getattr(inst, 'imm', 1)
+  is_rdna4 = _is_r4(inst)
   offset = inst.ioffset if is_rdna4 else getattr(inst, 'offset', 0)
   if cdna:
     if soe and imm: off_s = f"{decode_src(inst.soffset, cdna)} offset:0x{offset:x}"
@@ -302,7 +284,7 @@ def _disasm_smem(inst: SMEM) -> str:
     off = getattr(inst, 'ioffset', getattr(inst, 'offset', 0))
     if off >= 0x800000: off = off - 0x1000000
     off_s = f"0x{off:x}" if off > 255 else str(off)
-    soff_s = decode_src(inst.soffset, cdna) if inst.soffset != 124 else "null"
+    soff_s = decode_src(inst.soffset, cdna) if inst.soffset != 124 else ("m0" if cdna else "null")
     if 'pc_rel' in name: return f"{name} {off_s}, {soff_s}, {_unwrap(inst.sdata)}"
     return f"{name} {sbase_str}, {off_s}, {soff_s}, {_unwrap(inst.sdata)}"
   # Use get_field_bits for register count
@@ -318,34 +300,58 @@ def _disasm_smem(inst: SMEM) -> str:
     scope_names = ['SCOPE_CU', 'SCOPE_SE', 'SCOPE_DEV', 'SCOPE_SYS']
     mods = (f" th:{th_names[th]}" if th else "") + (f" scope:{scope_names[scope]}" if scope else "")
     return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}, {sbase_str}, {off_s}{mods}"
+  if 'discard' in name: return f"{name} {sbase_str}, {off_s}" + _mods((inst.glc, " glc"), (getattr(inst, 'dlc', 0), " dlc"))
+  if name in ('s_memrealtime', 's_memtime'): return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}"
   return f"{name} {_fmt_sdst(inst.sdata, dst_n, cdna)}, {sbase_str}, {off_s}" + _mods((inst.glc, " glc"), (getattr(inst, 'dlc', 0), " dlc"))
 
 def _disasm_flat(inst: FLAT) -> str:
-  name, cdna = inst.op_name.lower(), _is_cdna(inst)
+  name, cdna, r4 = inst.op_name.lower(), _is_cdna(inst), _is_r4(inst)
   acc = getattr(inst, 'acc', 0)
   reg_fn = _areg if acc else _vreg
-  seg = ['flat', 'scratch', 'global'][inst.seg] if inst.seg < 3 else 'flat'
+  if r4: seg = 'flat' if (cls_name:=inst.__class__.__name__) == 'VFLAT' else ('global' if cls_name == 'VGLOBAL' else 'scratch')
+  else: seg = ['flat', 'scratch', 'global'][inst.seg] if inst.seg < 3 else 'flat'
   instr = f"{seg}_{name.split('_', 1)[1] if '_' in name else name}"
-  off_val = inst.offset if seg == 'flat' else (inst.offset if inst.offset < 4096 else inst.offset - 8192)
+  # Global/scratch uses 13-bit signed offset
+  offset = inst.ioffset if r4 else inst.offset
+  if seg != 'flat':
+    if cdna:
+      # CDNA: bit 12 is sign bit but not in offset field
+      raw = int.from_bytes(inst.to_bytes(), 'little')
+      off_val = offset | ((raw >> 12) & 1) << 12  # get bit 12
+    else:
+      off_val = offset
+    off_val = off_val if off_val < 4096 else off_val - 8192  # sign extend 13-bit
+  else:
+    off_val = offset
   # Use get_field_bits: data for stores/atomics, d for loads
   regs = inst.canonical_op_regs
   w = regs.get('data', regs.get('d', 1)) if 'store' in name or 'atomic' in name else regs.get('d', 1)
   off_s = f" offset:{off_val}" if off_val else ""
-  if cdna: mods = f"{off_s}{' glc' if inst.sc0 else ''}{' slc' if inst.nt else ''}"
+  if cdna: mods = f"{off_s}{' sc0' if inst.sc0 else ''}{' nt' if inst.nt else ''}{' sc1' if getattr(inst, 'sc1', 0) else ''}"
+  elif r4: mods = f"{off_s}{' scope' if inst.scope else ''}{' th' if inst.th else ''}"
   else: mods = f"{off_s}{' glc' if inst.glc else ''}{' slc' if inst.slc else ''}{' dlc' if inst.dlc else ''}"
-  if seg == 'flat' or _unwrap(inst.saddr) == 0x7F: saddr_s = ""
-  elif _unwrap(inst.saddr) == 124: saddr_s = ", off"
+  if seg == 'flat': saddr_s = ""
+  elif _unwrap(inst.saddr) in (0x7F, 124): saddr_s = ", off"
   elif seg == 'scratch': saddr_s = f", {decode_src(inst.saddr, cdna)}"
   elif _unwrap(inst.saddr) in (SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS): saddr_s = f", {(SPECIAL_PAIRS_CDNA if cdna else SPECIAL_PAIRS)[_unwrap(inst.saddr)]}"
   elif t := _ttmp(inst.saddr, 2): saddr_s = f", {t}"
   else: saddr_s = f", {_sreg(inst.saddr, 2) if _unwrap(inst.saddr) < 106 else decode_src(_unwrap(inst.saddr), cdna)}"
   if 'addtid' in name: return f"{instr} {reg_fn(inst.data if 'store' in name else inst.vdst)}{saddr_s}{mods}"
-  if cdna: addr_w = 1 if seg == 'scratch' else 2
+  # RDNA4: vaddr instead of addr, vsrc instead of data
+  addr = inst.vaddr if r4 else inst.addr
+  data = inst.vsrc if r4 else inst.data
+  # load_lds_* instructions: vaddr, saddr (no vdst, data goes to LDS)
+  if 'load_lds' in name:
+    addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
+    addr_s = "off" if not inst.sve and seg == 'scratch' else _vreg(addr, addr_w)
+    return f"{instr} {addr_s}{saddr_s}{mods}"
+  if seg == 'flat': addr_w = 2  # flat always uses 64-bit vaddr
+  elif cdna: addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
   else: addr_w = 1 if seg == 'scratch' or (_unwrap(inst.saddr) not in (0x7F, 124)) else 2
-  addr_s = "off" if not inst.sve and seg == 'scratch' else _vreg(inst.addr, addr_w)
-  data_s, vdst_s = reg_fn(inst.data, w), reg_fn(inst.vdst, w // 2 if 'cmpswap' in name else w)
-  glc_or_sc0 = inst.sc0 if cdna else inst.glc
+  addr_s = "off" if not inst.sve and seg == 'scratch' else _vreg(addr, addr_w)
+  data_s, vdst_s = reg_fn(data, w), reg_fn(inst.vdst, w // 2 if 'cmpswap' in name else w)
   if 'atomic' in name:
+    glc_or_sc0 = inst.sc0 if cdna else inst.glc
     return f"{instr} {vdst_s}, {addr_s}, {data_s}{saddr_s if seg != 'flat' else ''}{mods}" if glc_or_sc0 else f"{instr} {addr_s}, {data_s}{saddr_s if seg != 'flat' else ''}{mods}"
   if 'store' in name: return f"{instr} {addr_s}, {data_s}{saddr_s}{mods}"
   return f"{instr} {reg_fn(inst.vdst, w)}, {addr_s}{saddr_s}{mods}"
@@ -379,13 +385,14 @@ def _disasm_ds(inst: DS) -> str:
     return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}, {d0}, {d1}{off2}{gds}"
   if 'write2' in name: return f"{name} {addr}, {d0}, {d1}{off2}{gds}"
   if 'read2' in name: return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}{off2}{gds}"
-  if 'load' in name: return f"{name} {reg_fn(inst.vdst)}{off}{gds}" if 'addtid' in name else f"{name} {dst}, {addr}{off}{gds}"
-  if 'store' in name and not _has(name, 'cmp', 'xchg'):
+  if 'xchg2' in name: return f"{name} {reg_fn(inst.vdst, regs.get('d', 1))}, {addr}, {d0}, {d1}{off2}{gds}"
+  if 'load' in name or ('read' in name and 'read2' not in name): return f"{name} {reg_fn(inst.vdst)}{off}{gds}" if 'addtid' in name else f"{name} {dst}, {addr}{off}{gds}"
+  if ('store' in name or 'write' in name) and not _has(name, 'cmp', 'xchg', 'write2'):
     return f"{name} {reg_fn(inst.data0)}{off}{gds}" if 'addtid' in name else f"{name} {addr}, {d0}{off}{gds}"
   if 'swizzle' in name or name == 'ds_ordered_count': return f"{name} {reg_fn(inst.vdst)}, {addr}{off}{gds}"
   if 'permute' in name: return f"{name} {reg_fn(inst.vdst)}, {addr}, {reg_fn(inst.data0)}{off}{gds}"
   if 'condxchg' in name: return f"{name} {reg_fn(inst.vdst, 2)}, {addr}, {reg_fn(inst.data0, 2)}{off}{gds}"
-  if _has(name, 'cmpstore', 'mskor', 'wrap'):
+  if _has(name, 'cmpst', 'mskor', 'wrap'):
     return f"{name} {dst}, {addr}, {d0}, {d1}{off}{gds}" if '_rtn' in name else f"{name} {addr}, {d0}, {d1}{off}{gds}"
   return f"{name} {dst}, {addr}, {d0}{off}{gds}" if '_rtn' in name else f"{name} {addr}, {d0}{off}{gds}"
 
@@ -458,17 +465,16 @@ def _disasm_vop3sd(inst: VOP3SD) -> str:
   return f"{name} {inst.vdst.fmt()}, {_fmt_sdst(inst.sdst, 1)}, {srcs}{' clamp' if clamp else ''}{_omod(inst.omod)}"
 
 def _disasm_vopd(inst: VOPD) -> str:
-  lit = inst._literal or getattr(inst, 'literal', None)
-  is_rdna4 = 'rdna4' in inst.__class__.__module__
-  op_enum = R4_VOPDOp if is_rdna4 else VOPDOp
-  vdst_y, nx, ny = (_unwrap(inst.vdsty) << 1) | ((_unwrap(inst.vdstx) & 1) ^ 1), op_enum(inst.opx).name.lower(), op_enum(inst.opy).name.lower()
+  lit = inst._literal
+  op_enum = R4_VOPDOp if _is_r4(inst) else VOPDOp
+  nx, ny = op_enum(inst.opx).name.lower(), op_enum(inst.opy).name.lower()
   def half(n, vd, s0, vs1):
     vd, vs1 = _vi(vd), _vi(vs1)
     if 'mov' in n: return f"{n} v{vd}, {_lit(inst, s0)}"
     if 'fmamk' in n and lit: return f"{n} v{vd}, {_lit(inst, s0)}, 0x{lit:x}, v{vs1}"
     if 'fmaak' in n and lit: return f"{n} v{vd}, {_lit(inst, s0)}, v{vs1}, 0x{lit:x}"
     return f"{n} v{vd}, {_lit(inst, s0)}, v{vs1}"
-  return f"{half(nx, inst.vdstx, inst.srcx0, inst.vsrcx1)} :: {half(ny, vdst_y, inst.srcy0, inst.vsrcy1)}"
+  return f"{half(nx, inst.vdstx, inst.srcx0, inst.vsrcx1)} :: {half(ny, inst.vdsty, inst.srcy0, inst.vsrcy1)}"
 
 def _disasm_vop3p(inst: VOP3P) -> str:
   name = inst.op_name.lower()
@@ -503,15 +509,21 @@ def _disasm_sop1(inst: SOP1) -> str:
     if 'swappc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2)}, {src}"
     if 'sendmsg_rtn' in name:
       v = _unwrap(inst.ssrc0)
-      msg_str = MSG.get(v)
+      try: msg_str = MSG(v).name if v != 255 else None  # MSG_RTN_ILLEGAL_MSG (255) not supported by LLVM
+      except ValueError: msg_str = None
       return f"{name} {_fmt_sdst(inst.sdst, dst_regs)}, sendmsg({msg_str})" if msg_str else f"{name} {_fmt_sdst(inst.sdst, dst_regs)}, 0x{v:x}"
-  sop1_src_only = ('S_ALLOC_VGPR', 'S_SLEEP_VAR', 'S_BARRIER_SIGNAL', 'S_BARRIER_SIGNAL_ISFIRST', 'S_BARRIER_INIT', 'S_BARRIER_JOIN')
+  sop1_src_only = ('S_ALLOC_VGPR', 'S_SLEEP_VAR', 'S_BARRIER_SIGNAL', 'S_BARRIER_SIGNAL_ISFIRST', 'S_BARRIER_INIT', 'S_BARRIER_JOIN', 'S_SET_GPR_IDX_IDX',
+                   'S_CBRANCH_JOIN')
   if inst.op_name in sop1_src_only: return f"{name} {src}"
+  if cdna:
+    if 'getpc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2, cdna)}"
+    if 'setpc_b64' in name or 'rfe_b64' in name: return f"{name} {src}"
+    if 'swappc_b64' in name: return f"{name} {_fmt_sdst(inst.sdst, 2, cdna)}, {src}"
   return f"{name} {_fmt_sdst(inst.sdst, dst_regs, cdna)}, {src}"
 
 def _disasm_sop2(inst: SOP2) -> str:
   cdna, name = _is_cdna(inst), inst.op_name.lower()
-  lit = getattr(inst, '_literal', None)
+  lit = inst._literal
   # Use get_field_bits for register sizes
   regs = inst.canonical_op_regs
   dn, s0n, s1n = regs['d'], regs['s0'], regs['s1']
@@ -520,21 +532,33 @@ def _disasm_sop2(inst: SOP2) -> str:
   dst = _fmt_sdst(inst.sdst, dn, cdna)
   if 'fmamk' in name and lit is not None: return f"{name} {dst}, {s0}, 0x{lit:x}, {s1}"
   if 'fmaak' in name and lit is not None: return f"{name} {dst}, {s0}, {s1}, 0x{lit:x}"
+  if name in ('s_cbranch_g_fork', 's_rfe_restore_b64'): return f"{name} {s0}, {s1}"  # no destination
   return f"{name} {dst}, {s0}, {s1}"
 
 def _disasm_sopc(inst: SOPC) -> str:
-  cdna, regs = _is_cdna(inst), inst.canonical_op_regs
+  cdna, regs, name = _is_cdna(inst), inst.canonical_op_regs, inst.op_name.lower()
   s0 = _lit(inst, inst.ssrc0) if _unwrap(inst.ssrc0) == 255 else _fmt_src(inst.ssrc0, regs['s0'], cdna)
+  if name == 's_set_gpr_idx_on':
+    imm = _unwrap(inst.ssrc1) & 0xf
+    flags = [n for i, n in enumerate(['SRC0', 'SRC1', 'SRC2', 'DST']) if imm & (1 << i)]
+    return f"{name} {s0}, gpr_idx({','.join(flags)})"
   s1 = _lit(inst, inst.ssrc1) if _unwrap(inst.ssrc1) == 255 else _fmt_src(inst.ssrc1, regs['s1'], cdna)
-  return f"{inst.op_name.lower()} {s0}, {s1}"
+  return f"{name} {s0}, {s1}"
 
+_HWREG_BLACKLIST = {'HW_REG_PC_LO', 'HW_REG_PC_HI', 'HW_REG_IB_DBG1', 'HW_REG_FLUSH_IB', 'HW_REG_SHADER_TBA_LO', 'HW_REG_SHADER_TBA_HI',
+                    'HW_REG_SHADER_FLAT_SCRATCH_LO', 'HW_REG_SHADER_FLAT_SCRATCH_HI', 'HW_REG_SHADER_CYCLES'}
+_HWREG_BLACKLIST_CDNA = {'HW_REG_PC_LO', 'HW_REG_PC_HI', 'HW_REG_IB_DBG1', 'HW_REG_FLUSH_IB', 'HW_REG_SQ_SHADER_TBA_LO', 'HW_REG_SQ_SHADER_TBA_HI',
+                         'HW_REG_SQ_SHADER_TMA_LO', 'HW_REG_SQ_SHADER_TMA_HI', 'HW_REG_SQ_PERF_SNAPSHOT_DATA', 'HW_REG_SQ_PERF_SNAPSHOT_DATA1',
+                         'HW_REG_SQ_PERF_SNAPSHOT_PC_LO', 'HW_REG_SQ_PERF_SNAPSHOT_PC_HI', 'HW_REG_XCC_ID'}
 def _disasm_sopk(inst: SOPK) -> str:
   op, name, cdna = inst.op, inst.op_name.lower(), _is_cdna(inst)
-  is_rdna4 = 'rdna4' in inst.__class__.__module__
-  hw = HWREG_RDNA4 if is_rdna4 else HWREG
+  is_rdna4 = _is_r4(inst)
+  hw = HWREG_CDNA if cdna else (HWREG_RDNA4 if is_rdna4 else HWREG)
+  blacklist = _HWREG_BLACKLIST_CDNA if cdna else _HWREG_BLACKLIST
   def fmt_hwreg(hid, hoff, hsz):
-    if hid not in hw: return f"0x{inst.simm16:x}"
-    hr_name = hw[hid]
+    try: hr_name = hw(hid).name.replace("HW_REG_WAVE_", "HW_REG_")
+    except ValueError: return f"0x{inst.simm16:x}"
+    if hr_name in blacklist: return f"0x{inst.simm16:x}"
     return f"hwreg({hr_name})" if hoff == 0 and hsz == 32 else f"hwreg({hr_name}, {hoff}, {hsz})"
   if name == 's_setreg_imm32_b32':
     hid, hoff, hsz = inst.simm16 & 0x3f, (inst.simm16 >> 6) & 0x1f, ((inst.simm16 >> 11) & 0x1f) + 1
@@ -552,14 +576,23 @@ def _disasm_vinterp(inst: VINTERP) -> str:
   mods = _mods((inst.waitexp, f"wait_exp:{inst.waitexp}"), (inst.clmp, "clamp"))
   return f"{inst.op_name.lower()} {inst.vdst.fmt()}, {_lit(inst, inst.src0, inst.neg & 1)}, {_lit(inst, inst.src1, inst.neg & 2)}, {_lit(inst, inst.src2, inst.neg & 4)}" + (" " + mods if mods else "")
 
-DISASM_HANDLERS: dict[type, callable] = {
-  VOP1: _disasm_vop1, VOP1_SDST: _disasm_vop1, VOP2: _disasm_vop2, VOPC: _disasm_vopc, VOP3: _disasm_vop3, VOP3_SDST: _disasm_vop3, VOP3SD: _disasm_vop3sd, VOPD: _disasm_vopd, VOP3P: _disasm_vop3p,
+DISASM_HANDLERS: dict[type, Callable[..., str]] = {
+  VOP1: _disasm_vop1, VOP1_SDST: _disasm_vop1, VOP1_SDST_LIT: _disasm_vop1, VOP1_LIT: _disasm_vop1,
+  VOP2: _disasm_vop2, VOP2_LIT: _disasm_vop2, VOPC: _disasm_vopc, VOPC_LIT: _disasm_vopc,
+  VOP3: _disasm_vop3, VOP3_SDST: _disasm_vop3, VOP3_SDST_LIT: _disasm_vop3, VOP3_LIT: _disasm_vop3, VOP3SD: _disasm_vop3sd, VOP3SD_LIT: _disasm_vop3sd,
+  VOPD: _disasm_vopd, VOPD_LIT: _disasm_vopd, VOP3P: _disasm_vop3p, VOP3P_LIT: _disasm_vop3p,
   VINTERP: _disasm_vinterp, SOPP: _disasm_sopp, SMEM: _disasm_smem, DS: _disasm_ds, FLAT: _disasm_flat, GLOBAL: _disasm_flat, SCRATCH: _disasm_flat,
-  SOP1: _disasm_sop1, SOP2: _disasm_sop2, SOPC: _disasm_sopc, SOPK: _disasm_sopk,
+  SOP1: _disasm_sop1, SOP1_LIT: _disasm_sop1, SOP2: _disasm_sop2, SOP2_LIT: _disasm_sop2,
+  SOPC: _disasm_sopc, SOPC_LIT: _disasm_sopc, SOPK: _disasm_sopk, SOPK_LIT: _disasm_sopk,
   # RDNA4
-  R4_VOP1: _disasm_vop1, R4_VOP1_SDST: _disasm_vop1, R4_VOP2: _disasm_vop2, R4_VOPC: _disasm_vopc, R4_VOP3: _disasm_vop3, R4_VOP3_SDST: _disasm_vop3, R4_VOP3SD: _disasm_vop3sd,
-  R4_VOPD: _disasm_vopd, R4_VOP3P: _disasm_vop3p, R4_VINTERP: _disasm_vinterp, R4_SOPP: _disasm_sopp, R4_SMEM: _disasm_smem,
-  R4_DS: _disasm_ds, R4_SOP1: _disasm_sop1, R4_SOP2: _disasm_sop2, R4_SOPC: _disasm_sopc, R4_SOPK: _disasm_sopk}
+  R4_VOP1: _disasm_vop1, R4_VOP1_SDST: _disasm_vop1, R4_VOP1_SDST_LIT: _disasm_vop1, R4_VOP1_LIT: _disasm_vop1,
+  R4_VOP2: _disasm_vop2, R4_VOP2_LIT: _disasm_vop2, R4_VOPC: _disasm_vopc, R4_VOPC_LIT: _disasm_vopc,
+  R4_VOP3: _disasm_vop3, R4_VOP3_SDST: _disasm_vop3, R4_VOP3_SDST_LIT: _disasm_vop3, R4_VOP3_LIT: _disasm_vop3,
+  R4_VOP3SD: _disasm_vop3sd, R4_VOP3SD_LIT: _disasm_vop3sd, R4_VOP3P: _disasm_vop3p, R4_VOP3P_LIT: _disasm_vop3p,
+  R4_FLAT: _disasm_flat, R4_GLOBAL: _disasm_flat, R4_SCRATCH: _disasm_flat,
+  R4_VOPD: _disasm_vopd, R4_VOPD_LIT: _disasm_vopd, R4_VINTERP: _disasm_vinterp, R4_SOPP: _disasm_sopp, R4_SMEM: _disasm_smem, R4_DS: _disasm_ds,
+  R4_SOP1: _disasm_sop1, R4_SOP1_LIT: _disasm_sop1, R4_SOP2: _disasm_sop2, R4_SOP2_LIT: _disasm_sop2,
+  R4_SOPC: _disasm_sopc, R4_SOPC_LIT: _disasm_sopc, R4_SOPK: _disasm_sopk, R4_SOPK_LIT: _disasm_sopk}
 
 def disasm(inst: Inst) -> str: return DISASM_HANDLERS[type(inst)](inst)
 
@@ -567,105 +600,308 @@ def disasm(inst: Inst) -> str: return DISASM_HANDLERS[type(inst)](inst)
 # CDNA DISASSEMBLER SUPPORT
 # ═══════════════════════════════════════════════════════════════════════════════
 
-try:
-  from extra.assembly.amd.autogen.cdna.ins import (VOP1 as CDNA_VOP1, VOP2 as CDNA_VOP2, VOPC as CDNA_VOPC, VOP3A, VOP3B, VOP3P as CDNA_VOP3P,
-    SOP1 as CDNA_SOP1, SOP2 as CDNA_SOP2, SOPC as CDNA_SOPC, SOPK as CDNA_SOPK, SOPP as CDNA_SOPP, SMEM as CDNA_SMEM, DS as CDNA_DS,
-    FLAT as CDNA_FLAT, VOP1Op as CDNA_VOP1Op, VOP2Op as CDNA_VOP2Op, VOPCOp as CDNA_VOPCOp)
+from extra.assembly.amd.autogen.cdna.ins import (VOP1 as CDNA_VOP1, VOP1_LIT as CDNA_VOP1_LIT,
+  VOP1_SDWA as CDNA_VOP1_SDWA, VOP1_DPP16 as CDNA_VOP1_DPP16,
+  VOP2 as CDNA_VOP2, VOP2_LIT as CDNA_VOP2_LIT, VOP2_SDWA as CDNA_VOP2_SDWA, VOP2_DPP16 as CDNA_VOP2_DPP16,
+  VOPC as CDNA_VOPC, VOPC_LIT as CDNA_VOPC_LIT, VOPC_SDWA_SDST as CDNA_VOPC_SDWA_SDST,
+  VOP3 as CDNA_VOP3, VOP3_SDST as CDNA_VOP3_SDST, VOP3SD as CDNA_VOP3SD, VOP3P as CDNA_VOP3P, VOP3P_MFMA as CDNA_VOP3P_MFMA, VOP3PX2 as CDNA_VOP3PX2,
+  SOP1 as CDNA_SOP1, SOP1_LIT as CDNA_SOP1_LIT, SOP2 as CDNA_SOP2, SOP2_LIT as CDNA_SOP2_LIT,
+  SOPC as CDNA_SOPC, SOPC_LIT as CDNA_SOPC_LIT, SOPK as CDNA_SOPK, SOPK_LIT as CDNA_SOPK_LIT,
+  SOPP as CDNA_SOPP, SMEM as CDNA_SMEM, DS as CDNA_DS,
+  FLAT as CDNA_FLAT, GLOBAL as CDNA_GLOBAL, SCRATCH as CDNA_SCRATCH, MUBUF as CDNA_MUBUF)
 
-  def _cdna_src(inst, v, neg, abs_=0, n=1):
-    s = _lit(inst, v) if v == 255 else _fmt_src(v, n, cdna=True)
-    if abs_: s = f"|{s}|"
-    return f"neg({s})" if neg and v == 255 else (f"-{s}" if neg else s)
+def _cdna_src(inst, v, neg, abs_=0, n=1):
+  s = _lit(inst, v) if v == 255 else _fmt_src(v, n, cdna=True)
+  if abs_: s = f"|{s}|"
+  return f"neg({s})" if neg and v == 255 else (f"-{s}" if neg else s)
 
-  _CDNA_VOP3_ALIASES = {'v_fmac_f64': 'v_mul_legacy_f32', 'v_dot2c_f32_bf16': 'v_mac_f32'}
+_CDNA_VOP3_ALIASES = {'v_fmac_f64': 'v_mul_legacy_f32', 'v_dot2c_f32_bf16': 'v_mac_f32'}
 
-  def _disasm_vop3a(inst) -> str:
-    op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
-    name = inst.op_name.lower() or f'vop3a_op_{op_val}'
-    n = inst.num_srcs() or _num_srcs(inst)
-    cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
-    orig_name = name
-    name = _CDNA_VOP3_ALIASES.get(name, name)
-    if name != orig_name:
-      s0, s1 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, 1), _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, 1)
-      s2 = ""
-      dst = _vreg(inst.vdst)
-    else:
-      regs = inst.canonical_op_regs
-      dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
-      s0, s1, s2 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, r0), _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, r1), _cdna_src(inst, inst.src2, inst.neg&4, inst.abs&4, r2)
-      dst = _vreg(inst.vdst, dregs) if dregs > 1 else _vreg(inst.vdst)
-    if op_val >= 512:
-      return f"{name} {dst}, {s0}, {s1}, {s2}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{cl}{om}"
-    if op_val < 256:
-      sdst = _fmt_sdst(inst.vdst, 2, cdna=True)
-      return f"{name}_e64 {sdst}, {s0}, {s1}{cl}"
-    if 320 <= op_val < 512:
-      if name in ('v_nop', 'v_clrexcp'): return f"{name}_e64"
-      return f"{name}_e64 {dst}, {s0}{cl}{om}"
-    if name == 'v_cndmask_b32':
-      s2 = _fmt_src(inst.src2, 2, cdna=True)
-      return f"{name}_e64 {dst}, {s0}, {s1}, {s2}{cl}{om}"
-    if name in ('v_mul_legacy_f32', 'v_mac_f32'):
-      return f"{name}_e64 {dst}, {s0}, {s1}{cl}{om}"
-    suf = "_e64" if op_val < 512 else ""
-    return f"{name}{suf} {dst}, {s0}, {s1}, {s2}{cl}{om}" if n == 3 else f"{name}{suf} {dst}, {s0}, {s1}{cl}{om}"
-
-  def _disasm_vop3b(inst) -> str:
-    op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
-    name = inst.op_name.lower() or f'vop3b_op_{op_val}'
-    n = inst.num_srcs() or _num_srcs(inst)
+def _disasm_vop3a(inst) -> str:
+  op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
+  name = inst.op_name.lower() or f'vop3a_op_{op_val}'
+  n = inst.num_srcs() or _num_srcs(inst)
+  cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
+  # _sr_ instructions use 4-element op_sel (src2 for byte selection)
+  opsel_n = 3 if '_sr_' in name and n == 2 else n
+  opsel = _opsel_str(inst.opsel, opsel_n, inst.opsel != 0, False)
+  orig_name = name
+  name = _CDNA_VOP3_ALIASES.get(name, name)
+  if name != orig_name:
+    s0, s1 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, 1), _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, 1)
+    s2 = ""
+    dst = _vreg(inst.vdst)
+  else:
     regs = inst.canonical_op_regs
     dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
-    s0, s1, s2 = _cdna_src(inst, inst.src0, inst.neg&1, n=r0), _cdna_src(inst, inst.src1, inst.neg&2, n=r1), _cdna_src(inst, inst.src2, inst.neg&4, n=r2)
+    s0, s1, s2 = _cdna_src(inst, inst.src0, inst.neg&1, inst.abs&1, r0), _cdna_src(inst, inst.src1, inst.neg&2, inst.abs&2, r1), _cdna_src(inst, inst.src2, inst.neg&4, inst.abs&4, r2)
     dst = _vreg(inst.vdst, dregs) if dregs > 1 else _vreg(inst.vdst)
-    sdst = _fmt_sdst(inst.sdst, 2, cdna=True)
+  if op_val >= 512:
+    return f"{name} {dst}, {s0}, {s1}, {s2}{opsel}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{opsel}{cl}{om}"
+  if op_val < 256:
+    # VOPC: vdst is actually sdst (SGPR pair), but VGPRField adds 256 to the offset
+    sdst_val = _unwrap(inst.vdst)
+    if sdst_val >= 256: sdst_val -= 256
+    sdst = _fmt_sdst(sdst_val, 2, cdna=True)
+    return f"{name} {sdst}, {s0}, {s1}{cl}"
+  if 320 <= op_val < 512:
+    if name in ('v_nop', 'v_clrexcp', 'v_nop_e64', 'v_clrexcp_e64'): return name.replace('_e64', '')
+    return f"{name} {dst}, {s0}{cl}{om}"
+  if name == 'v_cndmask_b32':
+    s2 = _fmt_src(inst.src2, 2, cdna=True)
+    return f"{name} {dst}, {s0}, {s1}, {s2}{cl}{om}"
+  return f"{name} {dst}, {s0}, {s1}, {s2}{opsel}{cl}{om}" if n == 3 else f"{name} {dst}, {s0}, {s1}{opsel}{cl}{om}"
+
+def _disasm_vop3b(inst) -> str:
+  op_val = inst.op.value if hasattr(inst.op, 'value') else inst.op
+  name, cdna = inst.op_name.lower() or f'vop3b_op_{op_val}', _is_cdna(inst)
+  n = inst.num_srcs() or _num_srcs(inst)
+  regs = inst.canonical_op_regs
+  dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
+  s0, s1, s2 = _cdna_src(inst, inst.src0, inst.neg&1, n=r0), _cdna_src(inst, inst.src1, inst.neg&2, n=r1), _cdna_src(inst, inst.src2, inst.neg&4, n=r2)
+  # CDNA VOP3_SDST uses vdst field for sdst (but vdst adds 256), RDNA uses separate sdst field
+  sdst_val = getattr(inst, 'sdst', None)
+  if sdst_val is None and hasattr(inst, 'vdst'):
+    sdst_val = _unwrap(inst.vdst)
+    if sdst_val >= 256: sdst_val -= 256  # VGPRField adds 256, remove it for SGPR
+  # For CDNA VOP3_SDST (VOPC->VOP3), vdst is the scalar dest (sdst), there's no vdst output
+  if cdna and 'v_cmp' in name:
+    sdst = _fmt_sdst(sdst_val, 2, cdna=True)
     cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
-    if name in ('v_addc_co_u32', 'v_subb_co_u32', 'v_subbrev_co_u32'):
-      s2 = _fmt_src(inst.src2, 2, cdna=True)
-      return f"{name}_e64 {dst}, {sdst}, {s0}, {s1}, {s2}{cl}{om}"
-    suf = "_e64" if 'co_' in name else ""
-    return f"{name}{suf} {dst}, {sdst}, {s0}, {s1}, {s2}{cl}{om}" if n == 3 else f"{name}{suf} {dst}, {sdst}, {s0}, {s1}{cl}{om}"
+    return f"{name} {sdst}, {s0}, {s1}{cl}{om}"
+  dst = _vreg(inst.vdst, dregs) if dregs > 1 else _vreg(inst.vdst)
+  sdst = _fmt_sdst(sdst_val, 2, cdna=cdna)
+  cl, om = " clamp" if inst.clmp else "", _omod(inst.omod)
+  if name in ('v_addc_co_u32', 'v_subb_co_u32', 'v_subbrev_co_u32'):
+    s2 = _fmt_src(inst.src2, 2, cdna=cdna)
+  return f"{name} {dst}, {sdst}, {s0}, {s1}, {s2}{cl}{om}" if n == 3 else f"{name} {dst}, {sdst}, {s0}, {s1}{cl}{om}"
 
-  def _disasm_cdna_vop3p(inst) -> str:
-    name, n = inst.op_name.lower(), inst.num_srcs() or 2
-    is_mfma = 'mfma' in name or 'smfmac' in name
-    is_accvgpr = 'accvgpr' in name
-    get_src = lambda v, sc: _lit(inst, v) if v == 255 else _fmt_src(v, sc, cdna=True)
+def _disasm_cdna_vop3p(inst) -> str:
+  name, n = inst.op_name.lower(), inst.num_srcs() or 2
+  is_mfma = 'mfma' in name or 'smfmac' in name
+  is_accvgpr = 'accvgpr' in name
+  get_src = lambda v, sc: _lit(inst, v) if v == 255 else _fmt_src(v, sc, cdna=True)
 
-    # Handle accvgpr read/write (accumulator register operations)
-    if is_accvgpr:
-      src0_off = _unwrap(inst.src0)
-      vdst_off = _vi(inst.vdst)
-      if 'read' in name:
-        # v_accvgpr_read_b32 vN, aM - reads from accumulator to VGPR
-        return f"{name}_b32 v{vdst_off}, a{src0_off - 256 if src0_off >= 256 else src0_off}"
-      if 'write' in name:
-        # v_accvgpr_write_b32 aM, src - writes to accumulator from source
-        src = _lit(inst, inst.src0) if src0_off == 255 else (f"v{src0_off - 256}" if src0_off >= 256 else decode_src(src0_off, cdna=True))
-        return f"{name}_b32 a{vdst_off}, {src}"
+  # Handle accvgpr read/write (accumulator register operations)
+  if is_accvgpr:
+    src0_off = _unwrap(inst.src0)
+    vdst_off = _vi(inst.vdst)
+    if 'read' in name:
+      # v_accvgpr_read_b32 vN, aM - reads from accumulator to VGPR
+      return f"{name}_b32 v{vdst_off}, a{src0_off - 256 if src0_off >= 256 else src0_off}"
+    if 'write' in name:
+      # v_accvgpr_write_b32 aM, src - writes to accumulator from source
+      src = _lit(inst, inst.src0) if src0_off == 255 else (f"v{src0_off - 256}" if src0_off >= 256 else decode_src(src0_off, cdna=True))
+      return f"{name}_b32 a{vdst_off}, {src}"
 
-    # Handle MFMA instructions with accumulator destinations
-    if is_mfma:
-      sc = 2 if 'iu4' in name else 4 if 'iu8' in name or 'i4' in name else 8 if 'f16' in name or 'bf16' in name else 4
-      src0, src1, src2 = get_src(inst.src0, sc), get_src(inst.src1, sc), get_src(inst.src2, 16)
-      dst = _areg(inst.vdst, 16)  # MFMA uses accumulator registers
-      opsel_hi = inst.opsel_hi
-      mods = ([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else []) + ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi != 3 else []) + \
-             ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else []) + ([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if inst.clmp else [])
-      return f"{name} {dst}, {src0}, {src1}, {src2}{' ' + ' '.join(mods) if mods else ''}"
+  # Handle v_mfma_ld_scale_b32 - special 2-operand format: v_mfma_ld_scale_b32 src0, src1
+  if 'ld_scale' in name:
+    src0, src1 = get_src(inst.src0, 1), get_src(inst.src1, 1)
+    mods = ([_fmt_bits("op_sel", inst.opsel, 2)] if inst.opsel else []) + \
+           ([_fmt_bits("op_sel_hi", inst.opsel_hi, 2)] if inst.opsel_hi != 3 else [])
+    return f"{name} {src0}, {src1}{' ' + ' '.join(mods) if mods else ''}"
 
-    # Standard VOP3P instructions
-    src0, src1, src2, dst = get_src(inst.src0, 1), get_src(inst.src1, 1), get_src(inst.src2, 1), _vreg(inst.vdst)
-    opsel_hi = inst.opsel_hi  # CDNA VOP3P only has 2 bits for opsel_hi (no opsel_hi2)
-    opsel_hi_default = 3  # CDNA default is 0b11 (2 bits), not 0b111 like RDNA
-    mods = ([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else []) + ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi != opsel_hi_default else []) + \
-           ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else []) + ([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if inst.clmp else [])
-    return f"{name} {dst}, {src0}, {src1}, {src2}{' ' + ' '.join(mods) if mods else ''}" if n == 3 else f"{name} {dst}, {src0}, {src1}{' ' + ' '.join(mods) if mods else ''}"
+  # Handle MFMA instructions with accumulator destinations
+  if is_mfma:
+    regs = inst.canonical_op_regs
+    dregs, r0, r1, r2 = regs['d'], regs['s0'], regs['s1'], regs['s2']
+    # Infer register counts from instruction name if not in operands table (e.g., v_mfma_f32_32x32x4_xf32)
+    if dregs == 1:
+      if '32x32' in name: dregs, r0, r1, r2 = 16, 2, 2, 16
+      elif '16x16' in name: dregs, r0, r1, r2 = 4, 2, 2, 4
+    # MFMA reuses VOP3P fields differently: clmp -> acc_cd (dest is acc), opsel_hi -> acc (src1/src2 are acc)
+    # acc field (bits 60-59): bit 0 = src2 is acc (always for MFMA), bit 1 = src1 is acc
+    acc = inst.opsel_hi  # opsel_hi field maps to acc for MFMA
+    acc_cd = inst.clmp   # clmp field maps to acc_cd for MFMA (dest is accumulator)
+    is_smfmac = 'smfmac' in name  # SMFMAC has different operand semantics
+    # Format sources: src0 is always VGPR, src1/src2 depend on acc bits
+    def mfma_src(v, sc, is_acc):
+      v = _unwrap(v)
+      if v == 255: return _lit(inst, v)
+      if 128 <= v <= 208 or 240 <= v <= 248: return _lit(inst, v)
+      base = v - 256 if v >= 256 else v
+      if is_acc: return _areg(base, sc)
+      return _vreg(base, sc)
+    src0 = get_src(inst.src0, r0)  # src0 is always VGPR
+    src1 = mfma_src(inst.src1, r1, acc & 2)  # bit 1 = src1 is acc
+    # For SMFMAC, src2 is always a VGPR index (1 register), not accumulator
+    src2 = _vreg(inst.src2) if is_smfmac else mfma_src(inst.src2, r2, acc_cd)
+    dst = _areg(inst.vdst, dregs) if acc_cd else _vreg(inst.vdst, dregs)
+    # MFMA uses neg:[...] not neg_lo:[...], and doesn't support op_sel_hi or clamp
+    # Only f64 MFMA instructions support neg modifier
+    # f8f6f4 MFMA instructions support cbsz/blgp modifiers
+    mods = []
+    if 'f8f6f4' in name:
+      if inst.neg_hi: mods.append(f"cbsz:{inst.neg_hi}")
+      if inst.neg: mods.append(f"blgp:{inst.neg}")
+    elif inst.neg and 'f64' in name:
+      mods.append(_fmt_bits("neg", inst.neg, n))
+    return f"{name} {dst}, {src0}, {src1}, {src2}{' ' + ' '.join(mods) if mods else ''}"
 
-  DISASM_HANDLERS.update({CDNA_VOP1: _disasm_vop1, CDNA_VOP2: _disasm_vop2, CDNA_VOPC: _disasm_vopc,
-    CDNA_SOP1: _disasm_sop1, CDNA_SOP2: _disasm_sop2, CDNA_SOPC: _disasm_sopc, CDNA_SOPK: _disasm_sopk, CDNA_SOPP: _disasm_sopp,
-    CDNA_SMEM: _disasm_smem, CDNA_DS: _disasm_ds, CDNA_FLAT: _disasm_flat,
-    VOP3A: _disasm_vop3a, VOP3B: _disasm_vop3b, CDNA_VOP3P: _disasm_cdna_vop3p})
-except ImportError:
-  pass
+  # Standard VOP3P instructions
+  src0, src1, src2, dst = get_src(inst.src0, 1), get_src(inst.src1, 1), get_src(inst.src2, 1), _vreg(inst.vdst)
+  opsel_hi = inst.opsel_hi  # CDNA VOP3P only has 2 bits for opsel_hi (no opsel_hi2)
+  opsel_hi_default = 3  # CDNA default is 0b11 (2 bits), not 0b111 like RDNA
+  mods = ([_fmt_bits("op_sel", inst.opsel, n)] if inst.opsel else []) + ([_fmt_bits("op_sel_hi", opsel_hi, n)] if opsel_hi != opsel_hi_default else []) + \
+          ([_fmt_bits("neg_lo", inst.neg, n)] if inst.neg else []) + ([_fmt_bits("neg_hi", inst.neg_hi, n)] if inst.neg_hi else []) + (["clamp"] if inst.clmp else [])
+  return f"{name} {dst}, {src0}, {src1}, {src2}{' ' + ' '.join(mods) if mods else ''}" if n == 3 else f"{name} {dst}, {src0}, {src1}{' ' + ' '.join(mods) if mods else ''}"
+
+def _disasm_mubuf(inst) -> str:
+  name = inst.op_name.lower()
+  # Determine vdata register count from instruction name
+  nregs = 4 if 'xyzw' in name else 3 if 'xyz' in name else 2 if 'xy' in name or 'x2' in name or 'f64' in name or 'dwordx2' in name else 1
+  vdata = _vreg(inst.vdata, nregs)
+  vaddr = _vreg(inst.vaddr) if inst.offen or inst.idxen else None
+  srsrc = str(inst.srsrc)
+  soffset_val = _unwrap(inst.soffset)
+  soffset = f"s{soffset_val}" if soffset_val < 128 else "off"
+  offset = f" offset:{inst.offset}" if inst.offset else ""
+  offen = " offen" if inst.offen else ""
+  idxen = " idxen" if inst.idxen else ""
+  lds = " lds" if inst.lds else ""
+  sc0 = " sc0" if inst.sc0 else ""
+  sc1 = " sc1" if inst.sc1 else ""
+  nt = " nt" if inst.nt else ""
+  # Handle special cases
+  if name in ('buffer_wbl2', 'buffer_inv'):
+    return f"{name}{sc0}{sc1}"
+  if vaddr:
+    return f"{name} {vdata}, {vaddr}, {srsrc}, {soffset}{offen}{idxen}{offset}{sc0}{nt}{sc1}{lds}"
+  return f"{name} {vdata}, off, {srsrc}, {soffset}{offset}{sc0}{nt}{sc1}{lds}"
+
+_SDWA_SEL = {0: 'BYTE_0', 1: 'BYTE_1', 2: 'BYTE_2', 3: 'BYTE_3', 4: 'WORD_0', 5: 'WORD_1', 6: 'DWORD'}
+
+def _disasm_vop1_sdwa(inst) -> str:
+  name = inst.op_name.lower().replace('_e32', '')
+  regs = inst.canonical_op_regs
+  dst = _vreg(inst.vdst, regs['d'])
+  # When s0=1, vsrc0 is SGPR/constant (VGPRField adds 256, so subtract it back)
+  if inst.s0 == 0: src0 = _vreg(inst.vsrc0, regs['s0'])
+  else:
+    raw = _unwrap(inst.vsrc0) - 256  # VGPRField adds 256
+    src0 = decode_src(raw, cdna=True)  # handles SGPRs, constants, specials
+  src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
+  mods = []
+  if inst.clmp: mods.append("clamp")
+  if inst.omod == 1: mods.append("mul:2")
+  elif inst.omod == 2: mods.append("mul:4")
+  elif inst.omod == 3: mods.append("div:2")
+  mods.append(f"src0_sel:{src0_sel}")
+  return f"{name}_sdwa {dst}, {src0} {' '.join(mods)}"
+
+def _decode_dpp(dpp: int) -> str:
+  """Decode DPP control value to string."""
+  if dpp < 0x100: return f"quad_perm:[{dpp&3},{(dpp>>2)&3},{(dpp>>4)&3},{(dpp>>6)&3}]"
+  if 0x100 <= dpp <= 0x10f: return f"row_shl:{dpp & 0xf}"
+  if 0x110 <= dpp <= 0x11f: return f"row_shr:{dpp & 0xf}"
+  if 0x120 <= dpp <= 0x12f: return f"row_ror:{dpp & 0xf}"
+  if dpp == 0x130: return "wave_shl:1"
+  if dpp == 0x134: return "wave_rol:1"
+  if dpp == 0x138: return "wave_shr:1"
+  if dpp == 0x13c: return "wave_ror:1"
+  if dpp == 0x140: return "row_mirror"
+  if dpp == 0x141: return "row_half_mirror"
+  if dpp == 0x142: return "row_bcast:15"
+  if dpp == 0x143: return "row_bcast:31"
+  if 0x150 <= dpp <= 0x15f: return f"row_newbcast:{dpp & 0xf}"
+  if 0x160 <= dpp <= 0x16f: return f"row_share:{dpp & 0xf}"
+  if 0x170 <= dpp <= 0x17f: return f"row_xmask:{dpp & 0xf}"
+  return f"dpp:{dpp:#x}"
+
+def _disasm_vop1_dpp(inst) -> str:
+  name = inst.op_name.lower().replace('_e32', '')
+  regs = inst.canonical_op_regs
+  dst, src0 = _vreg(inst.vdst, regs['d']), _vreg(inst.vsrc0, regs['s0'])
+  dpp_str = _decode_dpp(inst.dpp)
+  mods = [dpp_str]
+  if inst.row_mask != 0xf: mods.append(f"row_mask:{inst.row_mask:#x}")
+  if inst.bank_mask != 0xf: mods.append(f"bank_mask:{inst.bank_mask:#x}")
+  if inst.bc: mods.append("bound_ctrl:1")
+  return f"{name}_dpp {dst}, {src0} {' '.join(mods)}"
+
+def _disasm_vop2_sdwa(inst) -> str:
+  name, cdna = inst.op_name.lower().replace('_e32', ''), _is_cdna(inst)
+  regs = inst.canonical_op_regs
+  dst = _vreg(inst.vdst, regs['d'])
+  # When s0/s1=1, vsrc is SGPR/constant (VGPRField adds 256, so subtract it back)
+  src0 = _vreg(inst.vsrc0, regs['s0']) if inst.s0 == 0 else decode_src(_unwrap(inst.vsrc0) - 256, cdna)
+  src1 = _vreg(inst.vsrc1, regs['s1']) if inst.s1 == 0 else decode_src(_unwrap(inst.vsrc1) - 256, cdna)
+  src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
+  src1_sel = _SDWA_SEL.get(inst.src1_sel, f'SEL{inst.src1_sel}')
+  mods = []
+  if inst.clmp: mods.append("clamp")
+  if inst.omod == 1: mods.append("mul:2")
+  elif inst.omod == 2: mods.append("mul:4")
+  elif inst.omod == 3: mods.append("div:2")
+  if inst.src0_sel != 6: mods.append(f"src0_sel:{src0_sel}")
+  if inst.src1_sel != 6: mods.append(f"src1_sel:{src1_sel}")
+  mods_str = ' '.join(mods) if mods else ""
+  # CDNA carry instructions and cndmask need vcc operands
+  if cdna and name in _VOP2_CARRY_OUT: return f"{name}_sdwa {dst}, vcc, {src0}, {src1} {mods_str}".strip()
+  if cdna and name in _VOP2_CARRY_INOUT: return f"{name}_sdwa {dst}, vcc, {src0}, {src1}, vcc {mods_str}".strip()
+  if cdna and name == 'v_cndmask_b32': return f"{name}_sdwa {dst}, {src0}, {src1}, vcc {mods_str}".strip()
+  return f"{name}_sdwa {dst}, {src0}, {src1} {mods_str}".strip()
+
+def _disasm_vop2_dpp(inst) -> str:
+  name, cdna = inst.op_name.lower().replace('_e32', ''), _is_cdna(inst)
+  regs = inst.canonical_op_regs
+  dst, src0, src1 = _vreg(inst.vdst, regs['d']), _vreg(inst.vsrc0, regs['s0']), _vreg(inst.vsrc1, regs['s1'])
+  dpp_str = _decode_dpp(inst.dpp)
+  mods = [dpp_str]
+  if inst.row_mask != 0xf: mods.append(f"row_mask:{inst.row_mask:#x}")
+  if inst.bank_mask != 0xf: mods.append(f"bank_mask:{inst.bank_mask:#x}")
+  if inst.bc: mods.append("bound_ctrl:1")
+  # CDNA carry instructions and cndmask need vcc operands
+  if cdna and name in _VOP2_CARRY_OUT: return f"{name}_dpp {dst}, vcc, {src0}, {src1} {' '.join(mods)}"
+  if cdna and name in _VOP2_CARRY_INOUT: return f"{name}_dpp {dst}, vcc, {src0}, {src1}, vcc {' '.join(mods)}"
+  if cdna and name == 'v_cndmask_b32': return f"{name}_dpp {dst}, {src0}, {src1}, vcc {' '.join(mods)}"
+  return f"{name}_dpp {dst}, {src0}, {src1} {' '.join(mods)}"
+
+def _disasm_vopc_sdwa(inst) -> str:
+  name = inst.op_name.lower().replace('_e32', '')
+  regs = inst.canonical_op_regs
+  sdst = _fmt_sdst(inst.sdst, 2, cdna=True)
+  src0 = _vreg(inst.vsrc0, regs['s0']) if getattr(inst, 's0', 0) == 0 else decode_src(_unwrap(inst.vsrc0) - 256, cdna=True)
+  src1 = _vreg(inst.vsrc1, regs['s1']) if getattr(inst, 's1', 0) == 0 else decode_src(_unwrap(inst.vsrc1) - 256, cdna=True)
+  src0_sel = _SDWA_SEL.get(inst.src0_sel, f'SEL{inst.src0_sel}')
+  src1_sel = _SDWA_SEL.get(inst.src1_sel, f'SEL{inst.src1_sel}')
+  mods = []
+  if inst.src0_sel != 6: mods.append(f"src0_sel:{src0_sel}")
+  if inst.src1_sel != 6: mods.append(f"src1_sel:{src1_sel}")
+  return f"{name}_sdwa {sdst}, {src0}, {src1} {' '.join(mods)}".strip()
+
+def _disasm_vop3px2(inst) -> str:
+  """VOP3PX2 disassembler for scaled MFMA instructions."""
+  name = inst.op_name.lower()
+  regs = inst.canonical_op_regs
+  dregs, r2 = regs['d'], regs['s2']
+  # F8F6F4 MFMA: CBSZ selects matrix A format, BLGP selects matrix B format
+  # VGPRs: FP8/BF8(0,1)=8, FP6/BF6(2,3)=6, FP4(4)=4
+  vgprs = {0: 8, 1: 8, 2: 6, 3: 6, 4: 4}
+  r0, r1 = vgprs.get(inst.cbsz, 8), vgprs.get(inst.blgp, 8)
+  def mfma_src(v, sc, is_acc):
+    v = _unwrap(v)
+    if v == 255: return _lit(inst, v)
+    base = v - 256 if v >= 256 else v
+    return _areg(base, sc) if is_acc else _vreg(base, sc)
+  src0, src1, src2 = mfma_src(inst.src0, r0, False), mfma_src(inst.src1, r1, inst.acc & 2), mfma_src(inst.src2, r2, inst.acc_cd)
+  dst = _areg(inst.vdst, dregs) if inst.acc_cd else _vreg(inst.vdst, dregs)
+  scale_src0, scale_src1 = _vreg(inst.scale_src0), _vreg(inst.scale_src1)
+  mods = []
+  if inst.opsel: mods.append(_fmt_bits("op_sel", inst.opsel, 3))
+  if inst.opsel_hi != 0: mods.append(_fmt_bits("op_sel_hi", inst.opsel_hi, 3))
+  if inst.neg: mods.append(_fmt_bits("neg", inst.neg, 3))
+  if inst.cbsz: mods.append(f"cbsz:{inst.cbsz}")
+  if inst.blgp: mods.append(f"blgp:{inst.blgp}")
+  return f"{name} {dst}, {src0}, {src1}, {src2}, {scale_src0}, {scale_src1}{' ' + ' '.join(mods) if mods else ''}"
+
+DISASM_HANDLERS.update({CDNA_VOP1: _disasm_vop1, CDNA_VOP1_LIT: _disasm_vop1,
+  CDNA_VOP1_SDWA: _disasm_vop1_sdwa, CDNA_VOP1_DPP16: _disasm_vop1_dpp,
+  CDNA_VOP2: _disasm_vop2, CDNA_VOP2_LIT: _disasm_vop2,
+  CDNA_VOP2_SDWA: _disasm_vop2_sdwa, CDNA_VOP2_DPP16: _disasm_vop2_dpp,
+  CDNA_VOPC: _disasm_vopc, CDNA_VOPC_LIT: _disasm_vopc, CDNA_VOPC_SDWA_SDST: _disasm_vopc_sdwa,
+  CDNA_SOP1: _disasm_sop1, CDNA_SOP1_LIT: _disasm_sop1, CDNA_SOP2: _disasm_sop2, CDNA_SOP2_LIT: _disasm_sop2,
+  CDNA_SOPC: _disasm_sopc, CDNA_SOPC_LIT: _disasm_sopc, CDNA_SOPK: _disasm_sopk, CDNA_SOPK_LIT: _disasm_sopk, CDNA_SOPP: _disasm_sopp,
+  CDNA_SMEM: _disasm_smem, CDNA_DS: _disasm_ds, CDNA_FLAT: _disasm_flat, CDNA_GLOBAL: _disasm_flat, CDNA_SCRATCH: _disasm_flat,
+  CDNA_VOP3: _disasm_vop3a, CDNA_VOP3_SDST: _disasm_vop3b, CDNA_VOP3SD: _disasm_vop3b, CDNA_VOP3P: _disasm_cdna_vop3p, CDNA_VOP3P_MFMA: _disasm_cdna_vop3p,
+  CDNA_MUBUF: _disasm_mubuf, CDNA_VOP3PX2: _disasm_vop3px2})

extra/assembly/amd/dsl.py

@@ -1,27 +1,24 @@
 # dsl.py - clean DSL for AMD assembly
+from typing import Any
 
 # ══════════════════════════════════════════════════════════════
 # Registers - unified src encoding space (0-511)
 # ══════════════════════════════════════════════════════════════
 
-def _reg_size(t: str | None) -> int: return {'b64': 2, 'f64': 2, 'u64': 2, 'i64': 2, 'b128': 4}.get(t, 1)
-
 class Reg:
-  _NAMES = {106: "VCC_LO", 107: "VCC_HI", 124: "NULL", 125: "M0", 126: "EXEC_LO", 127: "EXEC_HI",
+  # Register names vary by arch: RDNA has NULL@124/M0@125, CDNA has M0@124/reserved@125
+  # RDNA4 has DPP8@233, CDNA has SDWA@249/DPP@250/VCCZ@251/EXECZ@252
+  _NAMES = {102: "FLAT_SCRATCH_LO", 103: "FLAT_SCRATCH_HI", 104: "XNACK_MASK_LO", 105: "XNACK_MASK_HI",
+            106: "VCC_LO", 107: "VCC_HI", 124: "NULL", 125: "M0", 126: "EXEC_LO", 127: "EXEC_HI",
+            233: "DPP8", 234: "DPP8FI", 235: "SHARED_BASE", 236: "SHARED_LIMIT", 237: "PRIVATE_BASE", 238: "PRIVATE_LIMIT",
             240: "0.5", 241: "-0.5", 242: "1.0", 243: "-1.0", 244: "2.0", 245: "-2.0", 246: "4.0", 247: "-4.0",
-            248: "INV_2PI", 250: "DPP16", 253: "SCC", 255: "LIT"}
+            248: "INV_2PI", 249: "SDWA", 250: "DPP", 251: "VCCZ", 252: "EXECZ", 253: "SCC", 254: "SRC_LDS_DIRECT", 255: "LIT"}
   _PAIRS = {106: "VCC", 126: "EXEC"}
 
   def __init__(self, offset: int = 0, sz: int = 512, *, neg: bool = False, abs_: bool = False, hi: bool = False):
     self.offset, self.sz = offset, sz
     self.neg, self.abs_, self.hi = neg, abs_, hi
 
-  # TODO: remove these legacy aliases
-  @property
-  def count(self): return self.sz
-  @property
-  def idx(self): return self.offset
-
   def __hash__(self): return hash((self.offset, self.sz, self.neg, self.abs_, self.hi))
   def __getitem__(self, key):
     if isinstance(key, slice):
@@ -78,9 +75,13 @@ EXEC = src[126:127]
 # 128: 0, 129-192: integers 1-64, 193-208: integers -1 to -16
 # 240-248: float constants (0.5, -0.5, 1.0, -1.0, 2.0, -2.0, 4.0, -4.0, 1/(2*PI))
 INV_2PI = src[248]
-DPP16 = src[250]
+SDWA = src[249]
+DPP = DPP16 = src[250]
+VCCZ = src[251]
+EXECZ = src[252]
 SCC = src[253]
-# 255: literal constant
+SRC_LDS_DIRECT = src[254]
+LIT = src[255]           # literal constant marker
 v = src[256:511]         # VGPR0-255
 
 # ══════════════════════════════════════════════════════════════
@@ -93,12 +94,13 @@ class _Bits:
 bits = _Bits()
 
 class BitField:
+  name: str | None
   def __init__(self, hi: int, lo: int, default: int = 0):
     self.hi, self.lo, self.default, self.name, self.mask = hi, lo, default, None, (1 << (hi - lo + 1)) - 1
-  def __set_name__(self, owner, name): self.name = name
-  def __eq__(self, other) -> 'FixedBitField':
+  def __set_name__(self, owner, name: str): self.name = name
+  def __eq__(self, other) -> 'FixedBitField':  # type: ignore[override]
     if isinstance(other, int): return FixedBitField(self.hi, self.lo, other)
-    return NotImplemented
+    raise TypeError(f"BitField.__eq__ expects int, got {type(other).__name__}")
   def enum(self, enum_cls) -> 'EnumBitField': return EnumBitField(self.hi, self.lo, enum_cls)
   def encode(self, val) -> int:
     assert isinstance(val, int), f"BitField.encode expects int, got {type(val).__name__}"
@@ -107,11 +109,14 @@ class BitField:
   def set(self, raw: int, val) -> int:
     if val is None: val = self.default
     encoded = self.encode(val)
+    # Handle signed values: convert negative to 2's complement
+    if encoded < 0: encoded = encoded & self.mask
     if encoded < 0 or encoded > self.mask: raise RuntimeError(f"field '{self.name}': value {encoded} doesn't fit in {self.hi - self.lo + 1} bits")
     return (raw & ~(self.mask << self.lo)) | (encoded << self.lo)
   def __get__(self, obj, objtype=None):
     if obj is None: return self
     return self.decode((obj._raw >> self.lo) & self.mask)
+  def __set__(self, obj, val): obj._raw = self.set(obj._raw, val)
 
 class FixedBitField(BitField):
   def set(self, raw: int, val=None) -> int:
@@ -119,11 +124,14 @@ class FixedBitField(BitField):
     return super().set(raw, self.default)
 
 class EnumBitField(BitField):
-  def __init__(self, hi: int, lo: int, enum_cls):
+  def __init__(self, hi: int, lo: int, enum_cls, allowed: set | None = None):
     super().__init__(hi, lo)
     self._enum = enum_cls
+    self.allowed = allowed  # if set, only these enum values are valid for this encoding
   def encode(self, val) -> int:
     if not isinstance(val, self._enum): raise RuntimeError(f"expected {self._enum.__name__}, got {type(val).__name__}")
+    if self.allowed is not None and val not in self.allowed:
+      raise RuntimeError(f"opcode {val.name} not allowed in this encoding")
     return val.value
   def decode(self, raw): return self._enum(raw)
 
@@ -165,7 +173,10 @@ class SrcField(BitField):
     # Resize register based on operand info (skip non-resizable special registers)
     # VCC/EXEC pairs (106, 126), NULL (124), M0 (125), float constants (240-255)
     if reg.offset not in (124, 125) and not 240 <= reg.offset <= 255:
-      if sz := obj.op_regs.get(self.name, 1): reg = Reg(reg.offset, sz, neg=reg.neg, abs_=reg.abs_, hi=reg.hi)
+      # Map variant field names (vsrc0->src0, vsrc1->src1, etc.) for DPP/SDWA classes
+      assert self.name is not None
+      name = self.name[1:] if self.name.startswith('v') and self.name[1:] in obj.op_regs else self.name
+      if sz := obj.op_regs.get(name, 1): reg = Reg(reg.offset, sz, neg=reg.neg, abs_=reg.abs_, hi=reg.hi)
     return reg
 
 class VGPRField(SrcField):
@@ -208,7 +219,12 @@ class VDSTYField(BitField):
     if not isinstance(val, Reg): raise TypeError(f"VDSTYField requires Reg, got {type(val).__name__}")
     if not (256 <= val.offset < 512): raise ValueError(f"VDSTYField requires VGPR, got offset {val.offset}")
     return (val.offset - 256) >> 1
-  def decode(self, raw): return raw  # raw value, actual vdsty = (raw << 1) | ((vdstx & 1) ^ 1)
+  def __get__(self, obj, objtype=None):
+    if obj is None: return self
+    raw = (obj._raw >> self.lo) & self.mask
+    vdstx_bit0 = (obj.vdstx.offset - 256) & 1
+    vgpr_idx = (raw << 1) | (vdstx_bit0 ^ 1)
+    return Reg(256 + vgpr_idx, 1)
 
 # ══════════════════════════════════════════════════════════════
 # Operand info from XML
@@ -224,6 +240,28 @@ OPERANDS = {**OPERANDS_CDNA, **OPERANDS_RDNA3, **OPERANDS_RDNA4}
 # Inst base class
 # ══════════════════════════════════════════════════════════════
 
+def _needs_literal(val) -> bool:
+  """Check if a value needs a literal constant (can't be encoded inline)."""
+  if val is None or isinstance(val, Reg): return False
+  if isinstance(val, float): return val not in SrcField._FLOAT_ENC
+  if isinstance(val, int): return not (0 <= val <= 64 or -16 <= val < 0)
+  return False
+
+def _get_variant(cls, suffix: str):
+  """Get a variant class by suffix (e.g., '_LIT') via module lookup."""
+  import sys
+  module = sys.modules.get(cls.__module__)
+  return getattr(module, f"{cls.__name__}{suffix}", None) if module else None
+
+def _canonical_name(name: str) -> str | None:
+  """Map operand name to canonical name."""
+  if name in ('src0', 'vsrc0', 'ssrc0'): return 's0'
+  if name in ('src1', 'vsrc1', 'ssrc1'): return 's1'
+  if name == 'src2': return 's2'
+  if name in ('vdst', 'sdst', 'sdata'): return 'd'
+  if name in ('data', 'vdata', 'data0', 'vsrc'): return 'data'
+  return None
+
 class Inst:
   _fields: list[tuple[str, BitField]]
   _base_size: int
@@ -238,18 +276,31 @@ class Inst:
     cls._fields = list(inherited.items())
     cls._base_size = (max(f.hi for _, f in cls._fields) + 8) // 8
 
+  def __new__(cls, *args, **kwargs):
+    # Auto-upgrade to variant if needed (only for base classes, not variants)
+    if not any(cls.__name__.endswith(sfx) for sfx in ('_LIT', '_DPP16', '_DPP8', '_SDWA', '_SDWA_SDST', '_MFMA')):
+      args_iter = iter(args)
+      for name, field in cls._fields:
+        if isinstance(field, FixedBitField): continue
+        val = kwargs.get(name) if name in kwargs else next(args_iter, None)
+        if not isinstance(field, SrcField): continue
+        if isinstance(val, Reg) and val.offset == 255 and (lit_cls := _get_variant(cls, '_LIT')): return lit_cls(*args, **kwargs)
+        if isinstance(val, Reg) and val.offset == 249:
+          if (sdwa_cls := _get_variant(cls, '_SDWA') or _get_variant(cls, '_SDWA_SDST')): return sdwa_cls(*args, **kwargs)
+        if isinstance(val, Reg) and val.offset == 250 and (dpp_cls := _get_variant(cls, '_DPP16')): return dpp_cls(*args, **kwargs)
+        if _needs_literal(val) and (lit_cls := _get_variant(cls, '_LIT')): return lit_cls(*args, **kwargs)
+    return object.__new__(cls)
+
   def __init__(self, *args, **kwargs):
     self._raw = 0
-    self._literal: int | None = kwargs.pop('literal', None)
     # Map positional args to field names (skip FixedBitFields)
     args_iter = iter(args)
-    vals = {}
+    vals: dict[str, Any] = {}
     for name, field in self._fields:
       if isinstance(field, FixedBitField): vals[name] = None
       elif name in kwargs: vals[name] = kwargs[name]
       else: vals[name] = next(args_iter, None)
-    remaining = list(args_iter)
-    assert not remaining, f"too many positional args: {remaining}"
+    assert not (remaining := list(args_iter)), f"too many positional args: {remaining}"
     # Extract modifiers from Reg objects and merge into neg/abs/opsel
     neg_bits, abs_bits, opsel_bits = 0, 0, 0
     for name, bit in [('src0', 0), ('src1', 1), ('src2', 2)]:
@@ -263,23 +314,27 @@ class Inst:
     if neg_bits: vals['neg'] = (vals.get('neg') or 0) | neg_bits
     if abs_bits: vals['abs'] = (vals.get('abs') or 0) | abs_bits
     if opsel_bits: vals['opsel'] = (vals.get('opsel') or 0) | opsel_bits
-    # Set all field values
+    # For _LIT classes, capture literal value from SrcFields that encode to 255
+    literal_val = None
     for name, field in self._fields:
       val = vals[name]
-      self._raw = field.set(self._raw, val)
-      # Capture literal for SrcFields that encoded to 255
-      if isinstance(field, SrcField) and val is not None and field.encode(val) + field._valid_range[0] == 255 and self._literal is None:
-        self._literal = _f32(val) if isinstance(val, float) else val & 0xFFFFFFFF
-    # Validate register sizes against operand info (skip special registers like NULL, VCC, EXEC)
+      if isinstance(field, SrcField) and val is not None and _needs_literal(val):
+        literal_val = _f32(val) if isinstance(val, float) else val & 0xFFFFFFFF
+    if literal_val is not None and 'literal' in vals:
+      vals['literal'] = literal_val
+    # Set all field values
+    for name, field in self._fields:
+      self._raw = field.set(self._raw, vals[name])
+    # Validate register sizes against operand info (skip special registers like NULL, VCC, EXEC, SDWA/DPP markers)
     for name, expected in self.op_regs.items():
       if (val := vals.get(name)) is None: continue
-      if isinstance(val, Reg) and val.sz != expected and not (106 <= val.offset <= 127 or val.offset == 253):
+      if isinstance(val, Reg) and val.sz != expected and not (106 <= val.offset <= 127 or 249 <= val.offset <= 255):
         raise TypeError(f"{name} expects {expected} register(s), got {val.sz}")
 
   @property
-  def op_name(self) -> str: return self.op.name
+  def op_name(self) -> str: return getattr(self, 'op').name
   @property
-  def operands(self) -> dict: return OPERANDS.get(self.op, {}) if hasattr(self, 'op') else {}
+  def operands(self) -> dict: return OPERANDS.get(getattr(self, 'op'), {}) if hasattr(self, 'op') else {}
   def _is_cdna(self) -> bool: return 'cdna' in type(self).__module__
 
   @functools.cached_property
@@ -291,17 +346,26 @@ class Inst:
     if not self._is_cdna():
       name = self.op_name.lower()
       if 'cndmask' in name and 'src2' in bits: bits['src2'] = 32
-      if '_co_ci_' in name:
-        if 'src2' in bits: bits['src2'] = 32
-        if 'sdst' in bits: bits['sdst'] = 32
+      if '_co_ci_' in name and 'src2' in bits: bits['src2'] = 32  # carry-in source
+      # VOP3SD: sdst is always wavefront-size dependent (carry-out or condition mask)
+      if 'VOP3SD' in type(self).__name__ and 'sdst' in bits: bits['sdst'] = 32
       if 'cmp' in name and 'vdst' in bits: bits['vdst'] = 32
     # GLOBAL/FLAT: addr is 32-bit if saddr is valid SGPR, 64-bit if saddr is NULL
-    if 'addr' in bits and (saddr_field := getattr(type(self), 'saddr', None)):
+    # SCRATCH: addr is always 32-bit (offset from scratch base, not absolute address)
+    if 'addr' in bits and (saddr_field := getattr(type(self), 'saddr', None)) and type(self).__name__ not in ('SCRATCH', 'VSCRATCH'):
       saddr_val = (self._raw >> saddr_field.lo) & saddr_field.mask  # access _raw directly to avoid recursion
       bits['addr'] = 64 if saddr_val in (124, 125) else 32  # 124=NULL, 125=M0
     # MUBUF/MTBUF: vaddr size depends on offen/idxen (1 or 2 regs)
     if 'vaddr' in bits and hasattr(self, 'offen') and hasattr(self, 'idxen'):
       bits['vaddr'] = max(1, self.offen + self.idxen) * 32
+    # F8F6F4 MFMA: CBSZ selects matrix A format, BLGP selects matrix B format
+    # VGPRs: FP8/BF8(0,1)=8, FP6/BF6(2,3)=6, FP4(4)=4
+    if 'f8f6f4' in getattr(self, 'op_name', '').lower():
+      # Use explicit fields if available (VOP3PX2), else extract from VOP3P-MAI bit positions
+      cbsz = getattr(self, 'cbsz') if hasattr(type(self), 'cbsz') else (self._raw >> 8) & 0x7
+      blgp = getattr(self, 'blgp') if hasattr(type(self), 'blgp') else (self._raw >> 61) & 0x7
+      vgprs = {0: 8, 1: 8, 2: 6, 3: 6, 4: 4}
+      bits['src0'], bits['src1'] = vgprs.get(cbsz, 8) * 32, vgprs.get(blgp, 8) * 32
     return bits
   @property
   def op_regs(self) -> dict[str, int]:
@@ -313,12 +377,17 @@ class Inst:
     """Get bit widths with canonical names: {'s0', 's1', 's2', 'd', 'data'}."""
     bits = {'d': 32, 's0': 32, 's1': 32, 's2': 32, 'data': 32}
     for name, val in self.op_bits.items():
-      if name in ('src0', 'vsrc0', 'ssrc0'): bits['s0'] = val
-      elif name in ('src1', 'vsrc1', 'ssrc1'): bits['s1'] = val
-      elif name == 'src2': bits['s2'] = val
-      elif name in ('vdst', 'sdst', 'sdata'): bits['d'] = val
-      elif name in ('data', 'vdata', 'data0'): bits['data'] = val
+      if (cn := _canonical_name(name)): bits[cn] = val
     return bits
+
+  @functools.cached_property
+  def canonical_operands(self) -> dict:
+    """Get operands with canonical names: {'s0', 's1', 's2', 'd', 'data'}."""
+    result = {}
+    for name, val in self.operands.items():
+      if (cn := _canonical_name(name)): result[cn] = val
+    return result
+
   @property
   def canonical_op_regs(self) -> dict[str, int]:
     """Get register counts with canonical names: {'s0', 's1', 's2', 'd', 'data'}."""
@@ -333,37 +402,45 @@ class Inst:
     return 0
   @classmethod
   def _size(cls) -> int: return cls._base_size
-  def size(self) -> int: return self._base_size + (4 if self._literal is not None else 0)
+  def size(self) -> int: return self._base_size
   def disasm(self) -> str:
     from extra.assembly.amd.disasm import disasm
     return disasm(self)
 
-  def to_bytes(self) -> bytes:
-    result = self._raw.to_bytes(self._base_size, 'little')
-    if self._literal is not None:
-      result += (self._literal & 0xFFFFFFFF).to_bytes(4, 'little')
-    return result
+  def to_bytes(self) -> bytes: return self._raw.to_bytes(self._base_size, 'little')
 
-  def has_literal(self) -> bool:
-    """Check if instruction has a 32-bit literal constant."""
+  @property
+  def _literal(self) -> int | None:
+    """Get the literal value if this instruction has one."""
+    return getattr(self, 'literal', None)
+
+  def _variant_suffix(self) -> str | None:
+    """Check if instruction needs a variant class (_LIT, _DPP8, _DPP16, _SDWA). Returns suffix or None."""
+    cls_name = type(self).__name__
+    # Don't check for variants if we're already a variant class
+    if any(s in cls_name for s in ('_LIT', '_DPP8', '_DPP16', '_SDWA')): return None
+    # VOPD: FMAMK/FMAAK opcodes always require literal (check by name since enum may differ across archs)
+    for name in ('opx', 'opy'):
+      if hasattr(self, name) and any(x in getattr(self, name).name for x in ('FMAMK', 'FMAAK')): return '_LIT'
     for name, field in self._fields:
-      if isinstance(field, SrcField) and getattr(self, name).offset == 255:
-        return True
-    # Check op, opx, opy for instructions that always have literals
-    for attr in ('op', 'opx', 'opy'):
-      if hasattr(self, attr) and any(x in getattr(self, attr).name for x in ('FMAMK', 'FMAAK', 'MADMK', 'MADAK', 'SETREG_IMM32')):
-        return True
-    return False
+      if isinstance(field, SrcField):
+        off = getattr(self, name).offset
+        if off == 255: return '_LIT'
+        if off == 249: return '_SDWA' if self._is_cdna() else '_DPP8'
+        if off == 250: return '_DPP16'
+    return None
 
   @classmethod
   def from_bytes(cls, data: bytes):
     inst = object.__new__(cls)
     inst._raw = int.from_bytes(data[:cls._base_size], 'little')
-    inst._literal = int.from_bytes(data[cls._base_size:cls._base_size + 4], 'little') if inst.has_literal() else None
+    # Upgrade to variant class if needed (_LIT, _DPP8, _DPP16, _SDWA)
+    if (suffix := inst._variant_suffix()) and (var_cls := _get_variant(cls, suffix)) is not None:
+      return var_cls.from_bytes(data)
     return inst
 
-  def __eq__(self, other): return type(self) is type(other) and self._raw == other._raw and self._literal == other._literal
-  def __hash__(self): return hash((type(self), self._raw, self._literal))
+  def __eq__(self, other): return type(self) is type(other) and self._raw == other._raw
+  def __hash__(self): return hash((type(self), self._raw))
 
   def __repr__(self):
     # collect (repr, is_default) pairs, strip trailing defaults so repr roundtrips with eval

extra/assembly/amd/generate.py

@@ -14,28 +14,39 @@ ARCHS = {
   "cdna": {"xml": "amdgpu_isa_cdna4.xml", "pdf": "https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/instruction-set-architectures/amd-instinct-cdna4-instruction-set-architecture.pdf"},
 }
 XML_URL = "https://gpuopen.com/download/machine-readable-isa/latest/"
-# Map XML encoding names to codebase names (arch-specific overrides in ARCH_NAME_MAP)
-NAME_MAP = {"VOP3_SDST_ENC": "VOP3SD", "VOPDXY": "VOPD", "VDS": "DS"}
-ARCH_NAME_MAP = {"cdna": {"VOP3": "VOP3A", "VOP3_SDST_ENC": "VOP3B"}}
+# Map XML encoding names to codebase names
+NAME_MAP = {"VOP3_SDST_ENC": "VOP3SD", "VOP3_SDST_ENC_LIT": "VOP3SD_LIT", "VOP3_SDST_ENC_DPP16": "VOP3SD_DPP16",
+            "VOP3_SDST_ENC_DPP8": "VOP3SD_DPP8", "VOPDXY": "VOPD", "VOPDXY_LIT": "VOPD_LIT", "VDS": "DS"}
 # Instructions missing from XML but present in PDF
 FIXES = {"rdna3": {"SOPK": {22: "S_SUBVECTOR_LOOP_BEGIN", 23: "S_SUBVECTOR_LOOP_END"}, "FLAT": {55: "FLAT_ATOMIC_CSUB_U32"}},
-         "rdna4": {"SOP1": {80: "S_GET_BARRIER_STATE", 81: "S_BARRIER_INIT", 82: "S_BARRIER_JOIN"}, "SOPP": {9: "S_WAITCNT", 21: "S_BARRIER_LEAVE"}}}
+         "rdna4": {"SOP1": {80: "S_GET_BARRIER_STATE", 81: "S_BARRIER_INIT", 82: "S_BARRIER_JOIN"}, "SOPP": {9: "S_WAITCNT", 21: "S_BARRIER_LEAVE"}},
+         "cdna": {"DS": {152: "DS_GWS_SEMA_RELEASE_ALL", 154: "DS_GWS_SEMA_V", 156: "DS_GWS_SEMA_P"},
+                  "VOP3P": {44: "V_MFMA_LD_SCALE_B32", 62: "V_MFMA_F32_16X16X8_XF32", 63: "V_MFMA_F32_32X32X4_XF32"}}}
+# Fields missing from XML but present in hardware (format: {arch: {encoding: [(name, hi, lo), ...]}})
+FIELD_FIXES = {"cdna": {"VOP3P": [("opsel_hi2", 14, 14)]}}
 # Encoding suffixes to strip (variants we don't generate separate classes for)
-_ENC_SUFFIXES = ("_INST_LITERAL", "_VOP_DPP16", "_VOP_DPP8", "_VOP_DPP", "_VOP_SDWA", "_NSA1", "_MFMA")
+_ENC_SUFFIXES = ("_NSA1",)
+# Encoding suffix to class suffix mapping (for variants we DO generate)
+_ENC_SUFFIX_MAP = {"_INST_LITERAL": "_LIT", "_VOP_DPP16": "_DPP16", "_VOP_DPP": "_DPP16", "_VOP_DPP8": "_DPP8",
+                   "_VOP_SDWA": "_SDWA", "_VOP_SDWA_SDST_ENC": "_SDWA_SDST", "_MFMA": "_MFMA"}
 # Field name normalization
 _FIELD_RENAMES = {"opsel_hi_2": "opsel_hi2", "op_sel_hi_2": "opsel_hi2", "op_sel": "opsel", "bound_ctrl": "bc",
-                  "tgt": "target", "row_en": "row", "unorm": "unrm", "clamp": "clmp", "wait_exp": "waitexp"}
-# Encoding variants to skip entirely
-_SKIP_ENCODINGS = ("LITERAL", "NSA", "DPP", "SDWA", "MFMA")
+                  "tgt": "target", "row_en": "row", "unorm": "unrm", "clamp": "clmp", "wait_exp": "waitexp",
+                  "simm32": "literal", "dpp_ctrl": "dpp", "acc_cd": "acc_cd", "acc": "acc",
+                  "dst_sel": "dst_sel", "dst_unused": "dst_unused", "src0_sel": "src0_sel", "src1_sel": "src1_sel"}
+# Encoding variants to skip entirely (NSA is for MIMG graphics instructions)
+_SKIP_ENCODINGS = ("NSA",)
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # XML parsing helpers
 # ═══════════════════════════════════════════════════════════════════════════════
 
 def _strip_enc(name: str) -> str:
-  """Strip ENC_ prefix and encoding variant suffixes."""
+  """Strip ENC_ prefix and normalize encoding suffixes."""
   name = name.removeprefix("ENC_")
   for sfx in _ENC_SUFFIXES: name = name.replace(sfx, "")
+  # Process longer suffixes first to avoid partial matches (e.g., _VOP_DPP8 before _VOP_DPP)
+  for old, new in sorted(_ENC_SUFFIX_MAP.items(), key=lambda x: -len(x[0])): name = name.replace(old, new)
   return name
 
 def _norm_field(name: str) -> str:
@@ -58,10 +69,16 @@ def _map_flat(enc_name: str, instr_name: str) -> str:
 # XML parsing
 # ═══════════════════════════════════════════════════════════════════════════════
 
-def parse_xml(filename: str, arch: str):
+def parse_xml(filename: str):
   root = ET.fromstring(zipfile.ZipFile(fetch(XML_URL)).read(filename))
-  name_map = {**NAME_MAP, **ARCH_NAME_MAP.get(arch, {})}
   encodings, enums, types, fmts, op_types_set = {}, {}, {}, {}, set()
+  # Extract HWREG and MSG enums from OperandTypes
+  op_enum_map = {("OPR_HWREG", "ID"): "HWREG", ("OPR_SENDMSG_RTN", "MSG"): "MSG"}
+  for ot in root.findall(".//OperandTypes/OperandType"):
+    ot_name = ot.findtext("OperandTypeName")
+    for field in ot.findall(".//Field"):
+      if (enum_name := op_enum_map.get((ot_name, field.findtext("FieldName")))):
+        enums[enum_name] = {int(pv.findtext("Value")): pv.findtext("Name").upper() for pv in field.findall(".//PredefinedValue")}
   # Extract DataFormats with BitCount
   for df in root.findall("ISA/DataFormats/DataFormat"):
     name, bits = df.findtext("DataFormatName"), df.findtext("BitCount")
@@ -69,36 +86,53 @@ def parse_xml(filename: str, arch: str):
   # Extract encoding definitions
   for enc in root.findall("ISA/Encodings/Encoding"):
     name = enc.findtext("EncodingName")
-    if not name.startswith("ENC_") and name not in ("VOP3_SDST_ENC", "VOPDXY"): continue
+    is_base = name.startswith("ENC_") or name in ("VOP3_SDST_ENC", "VOPDXY")
+    is_variant = any(sfx in name for sfx in _ENC_SUFFIX_MAP)
+    if not is_base and not is_variant: continue
     if any(s in name for s in _SKIP_ENCODINGS): continue
     fields = [(_norm_field(f.findtext("FieldName").lower()), int(f.find("BitLayout/Range").findtext("BitOffset") or 0) + int(f.find("BitLayout/Range").findtext("BitCount") or 0) - 1,
                int(f.find("BitLayout/Range").findtext("BitOffset") or 0))
               for f in enc.findall(".//MicrocodeFormat/BitMap/Field") if f.find("BitLayout/Range") is not None]
     ident = (enc.findall("EncodingIdentifiers/EncodingIdentifier") or [None])[0]
     enc_field = next((f for f in fields if f[0] == "encoding"), None)
-    enc_bits = "".join(ident.text[len(ident.text)-1-b] for b in range(enc_field[1], enc_field[2]-1, -1)) if ident is not None and enc_field else None
+    # For multi-dword formats, encoding field may be in higher dword but identifier pattern is always in dword0; use % 32
+    enc_bits = "".join(ident.text[len(ident.text)-1-b] for b in range(enc_field[1] % 32, (enc_field[2] % 32)-1, -1)) if ident is not None and enc_field else None
     base_name = _strip_enc(name)
-    encodings[name_map.get(base_name, base_name)] = (fields, enc_bits)
+    encodings[NAME_MAP.get(base_name, base_name)] = (fields, enc_bits)
   # Extract instruction opcodes and operand info
+  # Track which encodings each opcode appears in (for detecting LIT-only ops)
+  opcode_encs: dict[str, dict[int, set[str]]] = {}  # {base_fmt: {opcode: {enc_names}}}
   for instr in root.findall("ISA/Instructions/Instruction"):
     name = instr.findtext("InstructionName")
     for enc in instr.findall("InstructionEncodings/InstructionEncoding"):
       if enc.findtext("EncodingCondition") != "default": continue
       base, opcode = _map_flat(_strip_enc(enc.findtext("EncodingName")), name), int(enc.findtext("Opcode") or 0)
-      enc_name = name_map.get(base, base)
+      enc_name = NAME_MAP.get(base, base)
+      # Encoding variants use the same Op enum as the base format
+      base_enum = enc_name
+      for sfx in ("_SDWA_SDST", "_DPP16", "_DPP8", "_SDWA", "_LIT", "_MFMA"):
+        base_enum = base_enum.replace(sfx, "")
+      # Track which encodings this opcode appears in
+      opcode_encs.setdefault(base_enum, {}).setdefault(opcode, set()).add(enc_name)
       # ADDTID instructions go in both FLAT and GLOBAL enums (pcode uses FLATOp for these)
       if "ADDTID" in name:
         if base == "GLOBAL": enums.setdefault("FLAT", {})[opcode] = name
         elif base == "VGLOBAL": enums.setdefault("VFLAT", {})[opcode] = name
-      enums.setdefault(enc_name, {})[opcode] = name
+      enums.setdefault(base_enum, {})[opcode] = name
       # Extract operand info
       op_info = {op.findtext("FieldName").lower(): (op.findtext("DataFormatName"), int(op.findtext("OperandSize") or 0), op.findtext("OperandType"))
                  for op in enc.findall("Operands/Operand") if op.findtext("FieldName")}
       for fmt, _, otype in op_info.values():
         if fmt and fmt not in fmts: fmts[fmt] = 0
         if otype: op_types_set.add(otype)
-      if op_info: types[(name, enc_name)] = op_info
-  return encodings, enums, types, fmts, op_types_set
+      if op_info: types[(name, base_enum)] = op_info
+  # Find opcodes that only exist in a specific variant encoding (no base format version)
+  suffix_only_ops: dict[str, dict[str, set[int]]] = {}  # {suffix: {base_fmt: {opcodes}}}
+  for base_fmt, opcodes in opcode_encs.items():
+    for opcode, encs in opcodes.items():
+      suffix = next((s for s in _ENC_SUFFIX_MAP.values() if all(s in e for e in encs)), None)
+      if suffix is not None: suffix_only_ops.setdefault(suffix, {}).setdefault(base_fmt, set()).add(opcode)
+  return encodings, enums, types, fmts, op_types_set, suffix_only_ops
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # PDF parsing
@@ -188,6 +222,10 @@ def extract_pcode(pages: list[list[tuple[float, float, str, str]]], name_to_op:
 
 def write_common(all_fmts, all_op_types, path):
   lines = ["# autogenerated from AMD ISA XML - do not edit", "from enum import Enum, auto", ""]
+  lines.append("class ReprEnum(Enum):")
+  lines.append('  """Enum with clean repr that roundtrips with eval()."""')
+  lines.append('  def __repr__(self): return f"{type(self).__name__}.{self.name}"')
+  lines.append("")
   lines.append("class Fmt(Enum):")
   for fmt in sorted(all_fmts.keys()): lines.append(f"  {fmt} = auto()")
   lines.append("")
@@ -200,11 +238,11 @@ def write_common(all_fmts, all_op_types, path):
   with open(path, "w") as f: f.write("\n".join(lines))
 
 def write_enum(enums, path):
-  lines = ["# autogenerated from AMD ISA XML - do not edit", "from enum import Enum", "from extra.assembly.amd.autogen.common import Fmt, FMT_BITS, OpType  # noqa: F401", ""]
+  lines = ["# autogenerated from AMD ISA XML - do not edit", "from extra.assembly.amd.autogen.common import ReprEnum, Fmt, FMT_BITS, OpType  # noqa: F401", ""]
   for name, ops in sorted(enums.items()):
     if not ops: continue
     suffix = "_E32" if name in ("VOP1", "VOP2", "VOPC") else "_E64" if name == "VOP3" else ""
-    lines.append(f"class {name}Op(Enum):")
+    lines.append(f"class {name}(ReprEnum):" if name in ("HWREG", "MSG") else f"class {name}Op(ReprEnum):")
     aliases = []
     for op, mem in sorted(ops.items()):
       msuf = suffix if name != "VOP3" or op < 512 else ""
@@ -214,12 +252,17 @@ def write_enum(enums, path):
     lines.append("")
   with open(path, "w") as f: f.write("\n".join(lines))
 
-def write_ins(encodings, enums, arch, path):
-  _VGPR_FIELDS = {"vdst", "vdstx", "vsrc0", "vsrc1", "vsrc2", "vsrc3", "vsrcx1", "vsrcy1", "vaddr", "vdata", "data", "data0", "data1", "addr"}
+def write_ins(encodings, enums, suffix_only_ops, types, arch, path):
+  _VGPR_FIELDS = {"vdst", "vdstx", "vsrc0", "vsrc1", "vsrc2", "vsrc3", "vsrcx1", "vsrcy1", "vaddr", "vdata", "data", "data0", "data1", "addr", "vsrc"}
+  _VARIANT_SUFFIXES = ("_LIT", "_DPP16", "_DPP8", "_SDWA_SDST", "_SDWA", "_MFMA")
+  def get_base_fmt(fmt):
+    for sfx in _VARIANT_SUFFIXES: fmt = fmt.replace(sfx, "")
+    return fmt
   def field_def(name, hi, lo, fmt, enc_bits=None):
     bits = hi - lo + 1
+    base_fmt = get_base_fmt(fmt)
     if name == "encoding" and enc_bits: return f"FixedBitField({hi}, {lo}, 0b{enc_bits})"
-    if name == "op" and fmt not in ("DPP", "SDWA"): return f"EnumBitField({hi}, {lo}, {fmt}Op)"
+    if name == "op" and fmt not in ("DPP", "SDWA"): return f"EnumBitField({hi}, {lo}, {base_fmt}Op)"
     if name in ("opx", "opy"): return f"EnumBitField({hi}, {lo}, VOPDOp)"
     if name == "vdsty": return f"VDSTYField({hi}, {lo})"
     if name in _VGPR_FIELDS and bits == 8: return f"VGPRField({hi}, {lo})"
@@ -230,45 +273,135 @@ def write_ins(encodings, enums, arch, path):
     if name.startswith("ssrc") and bits == 8: return f"SSrcField({hi}, {lo})"
     if name in ("saddr", "soffset") and bits == 8: return f"SSrcField({hi}, {lo}, default=NULL)"
     if name.startswith("src") and bits == 9: return f"SrcField({hi}, {lo})"
-    if fmt == "VOP3P" and name == "opsel_hi": return f"BitField({hi}, {lo}, default=3)"
-    if fmt == "VOP3P" and name == "opsel_hi2": return f"BitField({hi}, {lo}, default=1)"
+    # GLOBAL/SCRATCH: offset is 13-bit signed [12:0], FLAT: 12-bit unsigned (XML has 12-bit for all)
+    if name == "offset" and base_fmt in ("GLOBAL", "SCRATCH"): return f"BitField(12, {lo})"
+    if base_fmt == "VOP3P" and name == "opsel_hi": return f"BitField({hi}, {lo}, default=3)"
+    if base_fmt == "VOP3P" and name == "opsel_hi2": return f"BitField({hi}, {lo}, default=1)"
     return f"BitField({hi}, {lo})"
   ORDER = ['encoding', 'op', 'opx', 'opy', 'vdst', 'vdstx', 'vdsty', 'sdst', 'vdata', 'sdata', 'addr', 'vaddr', 'data', 'data0', 'data1',
            'src0', 'srcx0', 'srcy0', 'vsrc0', 'ssrc0', 'src1', 'vsrc1', 'vsrcx1', 'vsrcy1', 'ssrc1', 'src2', 'vsrc2', 'src3', 'vsrc3',
-           'saddr', 'sbase', 'srsrc', 'ssamp', 'soffset', 'offset', 'simm16', 'en', 'target', 'attr', 'attr_chan',
+           'saddr', 'sbase', 'srsrc', 'ssamp', 'soffset', 'offset', 'simm16', 'literal', 'en', 'target', 'attr', 'attr_chan',
            'omod', 'neg', 'neg_hi', 'abs', 'clmp', 'opsel', 'opsel_hi', 'waitexp', 'wait_va',
-           'dmask', 'dim', 'seg', 'format', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe', 'unrm', 'done', 'row']
+           'dmask', 'dim', 'seg', 'format', 'offen', 'idxen', 'glc', 'dlc', 'slc', 'tfe', 'unrm', 'done', 'row',
+           'dpp', 'fi', 'bc', 'row_mask', 'bank_mask', 'src0_neg', 'src0_abs', 'src1_neg', 'src1_abs',
+           'cbsz', 'abid', 'acc_cd', 'acc', 'blgp', 'lane_sel_0', 'lane_sel_1', 'lane_sel_2', 'lane_sel_3',
+           'lane_sel_4', 'lane_sel_5', 'lane_sel_6', 'lane_sel_7', 'dst_sel', 'dst_unused', 'src0_sel', 'src1_sel']
   sort_fields = lambda fields: sorted(fields, key=lambda f: (ORDER.index(f[0]) if f[0] in ORDER else 999, f[2]))
 
+  # Separate base encodings from variants
+  base_encodings, variant_encodings = {}, {}
+  for enc_name, data in encodings.items():
+    base = get_base_fmt(enc_name)
+    if base == enc_name: base_encodings[enc_name] = data
+    else: variant_encodings[enc_name] = data
+
+  # Build sets of ops by their vdst type from operand metadata
+  sdst_opcodes = {}  # ops where vdst is OPR_SREG (writes to SGPR)
+  for fmt, ops in enums.items():
+    for op, name in ops.items():
+      op_types = types.get((name, fmt), {})
+      vdst_type = op_types.get("vdst", (None, None, None))[2]
+      if vdst_type == "OPR_SREG": sdst_opcodes.setdefault(fmt, set()).add(op)
+
   lines = ["# autogenerated from AMD ISA XML - do not edit", "# ruff: noqa: F401,F403",
            "from extra.assembly.amd.dsl import *", f"from extra.assembly.amd.autogen.{arch}.enum import *", "import functools", ""]
-  for enc_name, (fields, enc_bits) in sorted(encodings.items()):
-    if enc_name in ("FLAT", "VFLAT"):
+
+  def fmt_allowed(op_enum: str, ops: set[int]) -> str:
+    """Format allowed ops as {EnumName.MEMBER, ...}."""
+    names = [f"{op_enum}.{enums[op_enum.removesuffix('Op')][op]}" for op in sorted(ops)]
+    return "{" + ", ".join(names) + "}"
+
+  # Generate base classes first
+  for enc_name, (fields, enc_bits) in sorted(base_encodings.items()):
+    all_ops = set(enums.get(enc_name, {}).keys())
+    # Get suffix-only ops for this format (these can't be used in base class)
+    base_suffix_ops = set().union(*(d.get(enc_name, set()) for d in suffix_only_ops.values()))
+    # Exclude SDST ops from base class (they need VOP1_SDST/VOP3_SDST/VOP3B)
+    base_allowed = all_ops - base_suffix_ops - sdst_opcodes.get(enc_name, set())
+    # RDNA3 FLAT/GLOBAL/SCRATCH share encoding bits, differentiated by seg field
+    # RDNA4 VFLAT/VGLOBAL/VSCRATCH have distinct encoding bits, no seg field needed
+    has_seg_field = any(fn == "seg" for fn, _, _ in fields)
+    if enc_name in ("FLAT", "VFLAT") and has_seg_field:
       prefix = "V" if enc_name == "VFLAT" else ""
       for cls, seg, op_enum in [(f"{prefix}FLAT", 0, f"{prefix}FLATOp"), (f"{prefix}GLOBAL", 2, f"{prefix}GLOBALOp"), (f"{prefix}SCRATCH", 1, f"{prefix}SCRATCHOp")]:
+        cls_ops = set(enums.get(cls, {}).keys())
         lines.append(f"class {cls}(Inst):")
         for fn, hi, lo in sort_fields(fields):
           if fn == "seg": lines.append(f"  seg = FixedBitField({hi}, {lo}, {seg})")
-          elif fn == "op": lines.append(f"  op = EnumBitField({hi}, {lo}, {op_enum})")
+          elif fn == "op": lines.append(f"  op = EnumBitField({hi}, {lo}, {op_enum}, {fmt_allowed(op_enum, cls_ops)})")
           else: lines.append(f"  {fn} = {field_def(fn, hi, lo, cls, enc_bits)}")
         lines.append("")
-    elif enc_name not in ("FLAT_GLOBAL", "FLAT_SCRATCH", "FLAT_GLBL", "VGLOBAL", "VSCRATCH", "DPP", "SDWA"):
+    elif enc_name not in ("FLAT_GLOBAL", "FLAT_SCRATCH", "FLAT_GLBL", "DPP", "SDWA"):
       lines.append(f"class {enc_name}(Inst):")
       for fn, hi, lo in sort_fields(fields):
-        lines.append(f"  {fn} = {field_def(fn, hi, lo, enc_name, enc_bits if fn == 'encoding' else None)}")
+        if fn == "op":
+          base_fmt = get_base_fmt(enc_name)
+          lines.append(f"  op = EnumBitField({hi}, {lo}, {base_fmt}Op, {fmt_allowed(f'{base_fmt}Op', base_allowed)})")
+        else:
+          lines.append(f"  {fn} = {field_def(fn, hi, lo, enc_name, enc_bits if fn == 'encoding' else None)}")
       lines.append("")
-  # SDST variants
-  for base, field in [("VOP1", "vdst = SSrcField(24, 17)"), ("VOP3", "vdst = SSrcField(7, 0)")]:
-    if base in encodings: lines += [f"class {base}_SDST({base}):", f"  {field}", ""]
+
+  # Generate variant classes that inherit from base (only add extra fields)
+  for enc_name, (fields, enc_bits) in sorted(variant_encodings.items()):
+    base = get_base_fmt(enc_name)
+    if base not in base_encodings: continue  # skip if no base class
+    base_fields = {f[0] for f in base_encodings[base][0]}
+    extra_fields = [(fn, hi, lo) for fn, hi, lo in fields if fn not in base_fields]
+    # Check if this is a suffix-only variant
+    variant_suffix = next((sfx for sfx in _VARIANT_SUFFIXES if enc_name.endswith(sfx)), None)
+    is_suffix_variant = variant_suffix in suffix_only_ops
+    all_ops = set(enums.get(base, {}).keys())
+    if extra_fields or is_suffix_variant:
+      lines.append(f"class {enc_name}({base}):")
+      op_field = next((f for f in base_encodings[base][0] if f[0] == "op"), None)
+      # _LIT classes: override op to allow all opcodes (base excludes lit-only ops)
+      # other classes override op to only suffix-only opcodes
+      if op_field and is_suffix_variant:
+        _, hi, lo = op_field
+        allowed_ops = all_ops if variant_suffix == "_LIT" else suffix_only_ops[variant_suffix][base]
+        lines.append(f"  op = EnumBitField({hi}, {lo}, {base}Op, {fmt_allowed(f'{base}Op', allowed_ops)})")
+      for fn, hi, lo in sort_fields(extra_fields):
+        lines.append(f"  {fn} = {field_def(fn, hi, lo, enc_name)}")
+      lines.append("")
+
+  # SDST variants (special case - redefine vdst field type, restrict to SDST ops)
+  for base, field_hi, field_lo in [("VOP1", 24, 17), ("VOP3", 7, 0)]:
+    if base not in base_encodings: continue
+    sdst_ops = sdst_opcodes.get(base, set())
+    if not sdst_ops: continue
+    # For VOP3, all ops < 256 (compare/cmpx ops) use SDST encoding
+    all_base_ops = set(enums.get(base, {}).keys())
+    if base == "VOP3": sdst_ops = sdst_ops | {op for op in all_base_ops if op < 256}
+    op_field = next((f for f in base_encodings[base][0] if f[0] == "op"), None)
+    lines.append(f"class {base}_SDST({base}):")
+    if op_field:
+      _, hi, lo = op_field
+      lines.append(f"  op = EnumBitField({hi}, {lo}, {base}Op, {fmt_allowed(f'{base}Op', sdst_ops)})")
+    lines.append(f"  vdst = SSrcField({field_hi}, {field_lo})")
+    lines.append("")
+    # SDST_LIT class (for literals with SDST destination) - same ops, just adds literal field
+    lit_enc = variant_encodings.get(f"{base}_LIT")
+    if lit_enc:
+      lit_field = next((f for f in lit_enc[0] if f[0] == "literal"), None)
+      if lit_field:
+        lines.append(f"class {base}_SDST_LIT({base}_SDST):")
+        lines.append(f"  literal = BitField({lit_field[1]}, {lit_field[2]})")
+        lines.append("")
+
   # Instruction helpers
   lines.append("# instruction helpers")
-  SDST_OPS = {"V_READFIRSTLANE_B32", "V_READLANE_B32"}
   for fmt, ops in sorted(enums.items()):
-    if fmt not in encodings and fmt not in ("GLOBAL", "SCRATCH", "VGLOBAL", "VSCRATCH"): continue
+    if fmt not in base_encodings and fmt not in ("GLOBAL", "SCRATCH", "VGLOBAL", "VSCRATCH"): continue
     suffix = "_E32" if fmt in ("VOP1", "VOP2", "VOPC") else "_E64" if fmt == "VOP3" else ""
+    op_to_suffix = {op:suffix for suffix,ops in suffix_only_ops.items() for op in ops.get(fmt, set())}
+    fmt_sdst_ops = sdst_opcodes.get(fmt, set())
     for op, name in sorted(ops.items()):
       msuf = suffix if fmt != "VOP3" or op < 512 else ""
-      cls = "VOP1_SDST" if fmt == "VOP1" and name in SDST_OPS else "VOP3_SDST" if fmt == "VOP3" and (name in SDST_OPS or op < 256) else fmt
+      # Determine class: SDST variants, suffix-specific variants (e.g., _MFMA, _LIT), or base
+      if fmt == "VOP1" and op in fmt_sdst_ops: cls = "VOP1_SDST"
+      elif fmt == "VOP3" and (op in fmt_sdst_ops or op < 256): cls = "VOP3_SDST"
+      elif op_to_suffix.get(op): cls = f"{fmt}{op_to_suffix[op]}"
+      else: cls = fmt
       lines.append(f"{name.lower()}{msuf.lower()} = functools.partial({cls}, {fmt}Op.{name}{msuf})")
   with open(path, "w") as f: f.write("\n".join(lines))
 
@@ -316,9 +449,11 @@ if __name__ == "__main__":
   # First pass: parse XML for all architectures
   for arch, cfg in ARCHS.items():
     print(f"Parsing XML: {cfg['xml']} -> {arch}")
-    encodings, enums, types, fmts, op_types_set = parse_xml(cfg["xml"], arch)
+    encodings, enums, types, fmts, op_types_set, suffix_only_ops = parse_xml(cfg["xml"])
     for fmt, ops in FIXES.get(arch, {}).items(): enums.setdefault(fmt, {}).update(ops)
-    arch_data[arch] = {"encodings": encodings, "enums": enums, "types": types}
+    for fmt, fields in FIELD_FIXES.get(arch, {}).items():
+      if fmt in encodings: encodings[fmt] = (encodings[fmt][0] + fields, encodings[fmt][1])
+    arch_data[arch] = {"encodings": encodings, "enums": enums, "types": types, "suffix_only_ops": suffix_only_ops}
     for fmt, bits in fmts.items():
       assert fmt not in all_fmts or all_fmts[fmt] == bits, f"FMT_BITS mismatch for {fmt}: {all_fmts[fmt]} vs {bits}"
       all_fmts[fmt] = bits
@@ -331,7 +466,7 @@ if __name__ == "__main__":
   for arch, data in arch_data.items():
     base = pathlib.Path(__file__).parent / "autogen" / arch
     write_enum(data["enums"], base / "enum.py")
-    write_ins(data["encodings"], data["enums"], arch, base / "ins.py")
+    write_ins(data["encodings"], data["enums"], data["suffix_only_ops"], data["types"], arch, base / "ins.py")
     write_operands(data["types"], data["enums"], arch, base / "operands.py")
     print(f"  {arch}: {len(data['encodings'])} encodings, {sum(len(v) for v in data['enums'].values())} instructions")
   # Second pass: parse PDFs and write pcode

extra/assembly/amd/sqtt.py

@@ -26,7 +26,7 @@ class AluSrc(Enum):
   VALU_SALU = 3
 
 class InstOp(Enum):
-  """SQTT instruction operation types.
+  """SQTT instruction operation types for RDNA3 (gfx1100).
 
   Memory ops appear in two ranges depending on which SIMD executes them:
   - 0x1x-0x2x range: ops on traced SIMD
@@ -95,6 +95,21 @@ class InstOp(Enum):
   SALU_SAVEEXEC = 0x72    # s_*_saveexec_b32/b64
   VALU_CMPX = 0x73        # v_cmpx_*
 
+class InstOpL4(Enum):
+  """SQTT instruction operation types for RDNA4 (gfx1200). Different encoding from RDNA3."""
+  # TODO: we need to do discovery of all of these from instructions
+  SALU = 0x0
+  SMEM = 0x1
+  UNK_02 = 0x2
+  JUMP_NO = 0x4
+  UNK_06 = 0x6
+  VMEM = 0x10
+  UNK_11 = 0x11
+  VINTERP = 0x12
+  UNK_14 = 0x14
+  OTHER_VMEM = 0x5e
+  UNK_60 = 0x60
+
 # ═══════════════════════════════════════════════════════════════════════════════
 # PACKET TYPE BASE CLASS
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -117,9 +132,64 @@ class PacketType:
     return inst
 
   def __repr__(self) -> str:
-    fields_str = ", ".join(f"{k}={getattr(self, k)}" for k in self._fields if not k.startswith('_'))
+    fields_str = ", ".join(f"{k}={getattr(self, k)}" for k in self._fields if not k.startswith('_') and k != 'encoding')
     return f"{self.__class__.__name__}({fields_str})"
 
+# ═══════════════════════════════════════════════════════════════════════════════
+# TS PACKET TYPE DEFINITIONS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class TS_DELTA_S8_W3(PacketType):
+  encoding = bits[6:0] == 0b0100001
+  delta = bits[10:8]
+  _padding = bits[63:11]
+
+class TS_DELTA_S8_W3_L4(PacketType):  # Layout 4: 64->72 bits
+  encoding = bits[6:0] == 0b0100001
+  delta = bits[10:8]
+  _padding = bits[71:11]
+
+class TS_DELTA_S5_W3(PacketType):
+  encoding = bits[4:0] == 0b00110
+  delta = bits[7:5]
+  _padding = bits[51:8]
+
+class TS_DELTA_S5_W3_L4(PacketType):  # Layout 4: 52->56 bits
+  encoding = bits[4:0] == 0b00110
+  delta = bits[9:7]
+  _padding = bits[55:10]
+
+class TS_DELTA_SHORT(PacketType):
+  encoding = bits[3:0] == 0b1000
+  delta = bits[7:4]
+
+class TS_DELTA_OR_MARK(PacketType):
+  encoding = bits[6:0] == 0b0000001
+  delta = bits[47:12]
+  bit8 = bits[8:8]
+  bit9 = bits[9:9]
+  @property
+  def is_marker(self) -> bool: return bool(self.bit9 and not self.bit8)
+
+class TS_DELTA_OR_MARK_L4(PacketType):  # Layout 4: 48->64 bits
+  encoding = bits[6:0] == 0b0000001
+  delta = bits[63:12]
+  bit7 = bits[7:7]
+  bit8 = bits[8:8]
+  bit9 = bits[9:9]
+  @property
+  def is_marker(self) -> bool: return bool((self.bit9 and not self.bit8) or self.bit7)
+
+class TS_DELTA_S5_W2(PacketType):
+  encoding = bits[4:0] == 0b11100
+  delta = bits[6:5]
+  _padding = bits[47:7]
+
+class TS_DELTA_S5_W2_L4(PacketType):  # Layout 4: 48->40 bits
+  encoding = bits[4:0] == 0b11100
+  delta = bits[6:5]
+  _padding = bits[39:7]
+
 # ═══════════════════════════════════════════════════════════════════════════════
 # PACKET TYPE DEFINITIONS
 # ═══════════════════════════════════════════════════════════════════════════════
@@ -155,11 +225,6 @@ class WAVERDY(PacketType):  # exclude: 1 << 3
   delta = bits[7:5]
   mask = bits[23:8]
 
-class TS_DELTA_S8_W3(PacketType):
-  encoding = bits[6:0] == 0b0100001
-  delta = bits[10:8]
-  _padding = bits[63:11]
-
 class WAVEEND(PacketType):  # exclude: 1 << 4
   encoding = bits[4:0] == 0b10101
   delta = bits[7:5]
@@ -181,29 +246,36 @@ class WAVESTART(PacketType):  # exclude: 1 << 4
   @property
   def cu(self) -> int: return self.cu_lo | (self.flag7 << 3)
 
-class TS_DELTA_S5_W2(PacketType):
-  encoding = bits[4:0] == 0b11100
+class WAVESTART_L4(PacketType):  # Layout 4 has wave field at different position
+  encoding = bits[4:0] == 0b01100
   delta = bits[6:5]
-  _padding = bits[47:7]
+  flag7 = bits[7:7]
+  simd = bits[9:8]
+  cu_lo = bits[12:10]
+  wave = bits[19:15]
+  id7 = bits[31:20]
+  @property
+  def cu(self) -> int: return self.cu_lo | (self.flag7 << 3)
 
 class WAVEALLOC(PacketType):  # exclude: 1 << 10
   encoding = bits[4:0] == 0b00101
   delta = bits[7:5]
   _padding = bits[19:8]
 
-class TS_DELTA_S5_W3(PacketType):
-  encoding = bits[4:0] == 0b00110
+class WAVEALLOC_L4(PacketType):  # Layout 4: 20->24 bits
+  encoding = bits[4:0] == 0b00101
   delta = bits[7:5]
-  _padding = bits[51:8]
+  _padding = bits[23:8]
 
 class PERF(PacketType):  # exclude: 1 << 11
   encoding = bits[4:0] == 0b10110
   delta = bits[7:5]
   arg = bits[27:8]
 
-class TS_DELTA_SHORT(PacketType):
-  encoding = bits[3:0] == 0b1000
-  delta = bits[7:4]
+class PERF_L4(PacketType):  # Layout 4: 28->32 bits
+  encoding = bits[4:0] == 0b10110
+  delta = bits[9:7]
+  arg = bits[31:10]
 
 class NOP(PacketType):
   encoding = bits[3:0] == 0b0000
@@ -244,14 +316,6 @@ class SNAPSHOT(PacketType):
   delta = bits[9:7]
   snap = bits[63:10]
 
-class TS_DELTA_OR_MARK(PacketType):
-  encoding = bits[6:0] == 0b0000001
-  delta = bits[47:12]
-  bit8 = bits[8:8]
-  bit9 = bits[9:9]
-  @property
-  def is_marker(self) -> bool: return bool(self.bit9 and not self.bit8)
-
 class LAYOUT_HEADER(PacketType):
   encoding = bits[6:0] == 0b0010001
   delta = None  # type: ignore
@@ -271,56 +335,54 @@ class INST(PacketType):
   wave = bits[12:8]
   op = bits[19:13].enum(InstOp)
 
+class INST_L4(PacketType):  # Layout 4: different delta position and InstOp encoding
+  encoding = bits[2:0] == 0b010
+  delta = bits[5:3]
+  flag1 = bits[6:6]
+  flag2 = bits[7:7]
+  wave = bits[12:8]
+  op = bits[19:13].enum(InstOpL4)
+
 class UTILCTR(PacketType):
   encoding = bits[6:0] == 0b0110001
   delta = bits[8:7]
   ctr = bits[47:9]
 
-# All packet types in encoding priority order (more specific masks first, NOP last as fallback)
-PACKET_TYPES: list[type[PacketType]] = [
-  EVENT, EVENT_BIG,
-  TS_DELTA_S8_W3, TS_WAVE_STATE, SNAPSHOT, TS_DELTA_OR_MARK, LAYOUT_HEADER, UTILCTR,
-  IMMEDIATE_MASK, WAVERDY, WAVEEND, WAVESTART, TS_DELTA_S5_W2, WAVEALLOC, TS_DELTA_S5_W3, PERF,
-  VMEMEXEC, ALUEXEC, IMMEDIATE, TS_DELTA_SHORT, REG,
-  VALUINST, INST,
-  NOP,
-]
+# Packet types with rocprof type IDs as keys
+PACKET_TYPES_L3: dict[int, type[PacketType]] = {
+  1: VALUINST, 2: VMEMEXEC, 3: ALUEXEC, 4: IMMEDIATE, 5: IMMEDIATE_MASK, 6: WAVERDY, 7: TS_DELTA_S8_W3, 8: WAVEEND,
+  9: WAVESTART, 10: TS_DELTA_S5_W2, 11: WAVEALLOC, 12: TS_DELTA_S5_W3, 13: PERF, 14: UTILCTR, 15: TS_DELTA_SHORT,
+  16: NOP, 17: TS_WAVE_STATE, 18: EVENT, 19: EVENT_BIG, 20: REG, 21: SNAPSHOT, 22: TS_DELTA_OR_MARK, 23: LAYOUT_HEADER, 24: INST,
+}
+PACKET_TYPES_L4: dict[int, type[PacketType]] = {
+  **PACKET_TYPES_L3,
+  7: TS_DELTA_S8_W3_L4, 9: WAVESTART_L4, 10: TS_DELTA_S5_W2_L4, 11: WAVEALLOC_L4,
+  12: TS_DELTA_S5_W3_L4, 13: PERF_L4, 22: TS_DELTA_OR_MARK_L4, 24: INST_L4,
+}
+def _build_decode_tables(packet_types: dict[int, type[PacketType]]) -> tuple[dict[int, tuple], bytes]:
+  # Build state table: byte -> opcode. Sort by mask specificity (more bits first), NOP last
+  sorted_types = sorted(packet_types.items(), key=lambda x: (-bin(x[1].encoding.mask).count('1'), x[0] == 16))
+  state_table = bytes(next((op for op, cls in sorted_types if (b & cls.encoding.mask) == cls.encoding.default), 16) for b in range(256))
+  # Build decode info: opcode -> (pkt_cls, nib_count, delta_lo, delta_mask, special_case)
+  # special_case: 0=none, 1=TS_DELTA_OR_MARK (check is_marker), 2=TS_DELTA_SHORT (add 8)
+  decode_info = {}
+  for opcode, pkt_cls in packet_types.items():
+    delta_field = getattr(pkt_cls, 'delta', None)
+    special = {22: 1, 15: 2}.get(opcode, 0)  # TS_DELTA_OR_MARK=22, TS_DELTA_SHORT=15
+    decode_info[opcode] = (pkt_cls, pkt_cls._size_nibbles, delta_field.lo if delta_field else 0, delta_field.mask if delta_field else 0, special)
+  return decode_info, state_table
 
-def _build_state_table() -> tuple[bytes, dict[int, type[PacketType]]]:
-  table = [len(PACKET_TYPES) - 1] * 256  # default to NOP
-  opcode_to_class: dict[int, type[PacketType]] = {i: cls for i, cls in enumerate(PACKET_TYPES)}
-
-  for byte_val in range(256):
-    for opcode, pkt_cls in enumerate(PACKET_TYPES):
-      if (byte_val & pkt_cls.encoding.mask) == pkt_cls.encoding.default:
-        table[byte_val] = opcode
-        break
-
-  return bytes(table), opcode_to_class
-
-STATE_TO_OPCODE, OPCODE_TO_CLASS = _build_state_table()
-
-# Precompute special case opcodes
-_TS_DELTA_OR_MARK_OPCODE = next(op for op, cls in OPCODE_TO_CLASS.items() if cls is TS_DELTA_OR_MARK)
-_TS_DELTA_SHORT_OPCODE = next(op for op, cls in OPCODE_TO_CLASS.items() if cls is TS_DELTA_SHORT)
-
-# Combined lookup: opcode -> (pkt_cls, nib_count, delta_lo, delta_mask, special_case)
-# special_case: 0=none, 1=TS_DELTA_OR_MARK, 2=TS_DELTA_SHORT
-_DECODE_INFO: dict[int, tuple] = {}
-for _opcode, _pkt_cls in OPCODE_TO_CLASS.items():
-  _delta_field = getattr(_pkt_cls, 'delta', None)
-  _delta_lo = _delta_field.lo if _delta_field else 0
-  _delta_mask = _delta_field.mask if _delta_field else 0
-  _special = 1 if _opcode == _TS_DELTA_OR_MARK_OPCODE else (2 if _opcode == _TS_DELTA_SHORT_OPCODE else 0)
-  _DECODE_INFO[_opcode] = (_pkt_cls, _pkt_cls._size_nibbles, _delta_lo, _delta_mask, _special)
+_DECODE_INFO_L3, _STATE_TABLE_L3 = _build_decode_tables(PACKET_TYPES_L3)
+_DECODE_INFO_L4, _STATE_TABLE_L4 = _build_decode_tables(PACKET_TYPES_L4)
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # DECODER
 # ═══════════════════════════════════════════════════════════════════════════════
 
 def decode(data: bytes) -> Iterator[PacketType]:
-  """Decode raw SQTT blob, yielding packet instances."""
+  """Decode raw SQTT blob, yielding packet instances. Auto-detects layout from LAYOUT_HEADER."""
   n, reg, pos, nib_off, nib_count, time = len(data), 0, 0, 0, 16, 0
+  decode_info, state_table = _DECODE_INFO_L3, _STATE_TABLE_L3  # default to layout 3, will update after seeing LAYOUT_HEADER
 
   while pos + ((nib_count + nib_off + 1) >> 1) <= n:
     need = nib_count - nib_off
@@ -333,13 +395,20 @@ def decode(data: bytes) -> Iterator[PacketType]:
     # 3. if odd, read low nibble
     if (nib_off := need & 1): reg = (reg >> 4) | ((data[pos] & 0xF) << 60)
 
-    opcode = STATE_TO_OPCODE[reg & 0xFF]
-    pkt_cls, nib_count, delta_lo, delta_mask, special = _DECODE_INFO[opcode]
+    opcode = state_table[reg & 0xFF]
+    pkt_cls, nib_count, delta_lo, delta_mask, special = decode_info[opcode]
     delta = (reg >> delta_lo) & delta_mask
-    if special == 1 and (reg >> 9) & 1 and not (reg >> 8) & 1: delta = 0  # TS_DELTA_OR_MARK marker
+    if special == 1:  # TS_DELTA_OR_MARK
+      pkt = pkt_cls.from_raw(reg, 0)  # create packet to check is_marker
+      if pkt.is_marker: delta = 0
     elif special == 2: delta += 8  # TS_DELTA_SHORT
     time += delta
-    yield pkt_cls.from_raw(reg, time)
+    pkt = pkt_cls.from_raw(reg, time)
+    # detect layout from first LAYOUT_HEADER and switch decode tables if needed
+    # NOTE: CDNA uses a completely different 16-bit header format, not nibbles - not supported here
+    if pkt_cls is LAYOUT_HEADER and pkt.layout == 4:
+      decode_info, state_table = _DECODE_INFO_L4, _STATE_TABLE_L4
+    yield pkt
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # PRINTER
@@ -355,23 +424,26 @@ PACKET_COLORS = {
 def format_packet(p) -> str:
   from tinygrad.helpers import colored
   name = type(p).__name__
-  if isinstance(p, INST):
-    op_name = p.op.name if isinstance(p.op, InstOp) else f"0x{p.op:02x}"
+  if isinstance(p, (INST, INST_L4)):
+    op_name = p.op.name if isinstance(p.op, (InstOp, InstOpL4)) else f"0x{p.op:02x}"
     fields = f"wave={p.wave} op={op_name}" + (" flag1" if p.flag1 else "") + (" flag2" if p.flag2 else "")
   elif isinstance(p, VALUINST): fields = f"wave={p.wave}" + (" flag" if p.flag else "")
   elif isinstance(p, ALUEXEC): fields = f"src={p.src.name if isinstance(p.src, AluSrc) else p.src}"
   elif isinstance(p, VMEMEXEC): fields = f"src={p.src.name if isinstance(p.src, MemSrc) else p.src}"
-  elif isinstance(p, (WAVESTART, WAVEEND)): fields = f"wave={p.wave} simd={p.simd} cu={p.cu}"
+  elif isinstance(p, (WAVESTART, WAVESTART_L4, WAVEEND)): fields = f"wave={p.wave} simd={p.simd} cu={p.cu}"
   elif hasattr(p, '_fields'):
+    filt = {'delta', 'encoding'} if not isinstance(p, (TS_DELTA_OR_MARK, TS_DELTA_OR_MARK_L4)) else {'encoding'}
     fields = " ".join(f"{k}=0x{getattr(p, k):x}" if k in {'snap', 'val32'} else f"{k}={getattr(p, k)}"
-                      for k in p._fields if not k.startswith('_') and k not in {'delta', 'encoding'})
+                      for k in p._fields if not k.startswith('_') and k not in filt)
   else: fields = ""
-  return f"{p._time:8}: {colored(f'{name:18}', PACKET_COLORS.get(name, 'white'))} {fields}"
+  return f"{p._time:8}: {colored(f'{name:18}', PACKET_COLORS.get(name.replace('_L4', ''), 'white'))} {fields}"
 
 def print_packets(packets) -> None:
-  skip = {"NOP", "TS_DELTA_SHORT", "TS_WAVE_STATE", "TS_DELTA_OR_MARK", "TS_DELTA_S5_W2", "TS_DELTA_S5_W3", "TS_DELTA_S8_W3", "REG", "EVENT"}
+  from tinygrad.helpers import getenv
+  skip = {"NOP", "TS_DELTA_SHORT", "TS_WAVE_STATE", "TS_DELTA_OR_MARK",
+          "TS_DELTA_S5_W2", "TS_DELTA_S5_W3", "TS_DELTA_S8_W3", "REG", "EVENT"} if not getenv("NOSKIP") else {"NOP"}
   for p in packets:
-    if type(p).__name__ not in skip: print(format_packet(p))
+    if type(p).__name__.replace("_L4", "") not in skip: print(format_packet(p))
 
 if __name__ == "__main__":
   import sys, pickle

extra/assembly/amd/sqtt_cdna.py

@@ -0,0 +1,161 @@
+"""SQTT (SQ Thread Trace) packet decoder for CDNA/MI300 GPUs.
+
+CDNA uses a completely different 16-bit header format from RDNA's nibble-based encoding.
+"""
+from __future__ import annotations
+from typing import Iterator
+from extra.assembly.amd.dsl import bits
+from extra.assembly.amd.sqtt import PacketType
+
+# CDNA pkt_fmt -> size in bytes (extracted from rocprof hash table)
+CDNA_PKT_SIZES = {0: 2, 1: 8, 2: 8, 3: 4, 4: 2, 5: 6, 6: 2, 7: 2, 8: 2, 9: 2, 10: 2, 11: 8, 12: 6, 13: 4, 14: 8, 15: 6}
+
+class CDNA_DELTA(PacketType):
+  """pkt_fmt=0: 16-bit timestamp delta packet"""
+  encoding = bits[3:0] == 0
+  delta = bits[11:4]      # (data >> 4) & 0xff
+  unk_0 = bits[12:12]     # (data >> 0xc) & 1
+  unk_1 = bits[15:13]     # (data >> 0xd)
+
+class CDNA_TIMESTAMP(PacketType):
+  """pkt_fmt=1: 64-bit timestamp packet (case 0x0)"""
+  encoding = bits[3:0] == 1
+  unk_0 = bits[15:4]
+  timestamp = bits[63:16]   # stored as (data_word >> 0x10) in low 46 bits of local_58
+
+class CDNA_PKT_2(PacketType):
+  """pkt_fmt=2: 64-bit packet (case 0x4)"""
+  encoding = bits[3:0] == 2
+  unk_0 = bits[6:5]       # (data >> 5) & 3
+  unk_1 = bits[7:7]       # (data >> 7) + 1 & 1
+  unk_padding = bits[63:8]
+
+class CDNA_WAVESTART(PacketType):
+  """pkt_fmt=3: 32-bit WAVESTART packet (case 0x8)"""
+  encoding = bits[3:0] == 3
+  unk_0 = bits[5:5]       # (data >> 5) & 1
+  unk_1 = bits[9:6]       # (data >> 6) & 0xf
+  wave = bits[13:10]      # (data >> 10) & 0xf
+  simd = bits[15:14]      # (data >> 0xe) & 3
+  cu = bits[17:16]        # (data >> 0x10) & 3
+  unk_5 = bits[19:18]     # (data >> 0x12) & 3
+  unk_6 = bits[28:22]     # (data >> 0x16) & 0x7f
+  unk_padding = bits[31:29]
+
+class CDNA_PKT_4(PacketType):
+  """pkt_fmt=4: 16-bit packet (case 0xc, same as 0x8/0x14)"""
+  encoding = bits[3:0] == 4
+  unk_0 = bits[5:5]       # (data_word >> 5) & 1
+  unk_1 = bits[9:6]       # (data_word >> 6) & 0xf
+  unk_2 = bits[13:10]     # (data_word >> 10) & 0xf
+  unk_3 = bits[15:14]     # (data_word >> 0xe)
+
+class CDNA_PKT_5(PacketType):
+  """pkt_fmt=5: 48-bit packet (case 0x10)"""
+  encoding = bits[3:0] == 5
+  unk_0 = bits[6:5]       # (data >> 5) & 3
+  unk_1 = bits[7:7]       # (data >> 7) + 1 & 1
+  unk_2 = bits[15:9]      # (data >> 9) & 0x7f
+  unk_padding = bits[47:16]
+
+class CDNA_WAVEEND(PacketType):
+  """pkt_fmt=6: 16-bit WAVEEND packet (case 0x14, same as 0x8/0xc)"""
+  encoding = bits[3:0] == 6
+  unk_0 = bits[5:5]       # (data_word >> 5) & 1
+  unk_1 = bits[9:6]       # (data_word >> 6) & 0xf
+  wave = bits[13:10]      # (data_word >> 10) & 0xf
+  simd = bits[15:14]      # (data_word >> 0xe)
+
+class CDNA_EXEC(PacketType):
+  """pkt_fmt=10: 16-bit EXEC packet (case 0x24)"""
+  encoding = bits[3:0] == 10
+  unk_0 = bits[8:5]       # (data_word >> 5) & 0xf
+  unk_1 = bits[10:9]      # (data_word >> 9) & 3
+  unk_2 = bits[15:11]     # (data_word >> 0xb)
+
+class CDNA_PKT_11(PacketType):
+  """pkt_fmt=11: 64-bit packet (case 0x28)"""
+  encoding = bits[3:0] == 11
+  unk_0 = bits[8:5]       # (data_word >> 5) & 0xf
+  unk_1 = bits[10:9]      # (data_word >> 9) & 3
+  unk_2 = bits[15:15]     # (data_word >> 0xf) & 1
+  unk_padding = bits[63:16]
+
+class CDNA_INST(PacketType):
+  """pkt_fmt=13: 32-bit INST packet (case 0x30)"""
+  encoding = bits[3:0] == 13
+  unk_0 = bits[6:5]       # (data >> 5) & 3
+  unk_1 = bits[9:8]       # (data >> 8) & 3
+  unk_2 = bits[11:10]     # (data >> 10) & 3
+  unk_3 = bits[13:12]     # (data >> 0xc) & 3
+  unk_4 = bits[15:14]     # (data >> 0xe) & 3
+  unk_5 = bits[19:18]     # (data >> 0x12) & 3
+  unk_6 = bits[21:20]     # (data >> 0x14) & 3
+  unk_7 = bits[23:22]     # (data >> 0x16) & 3
+  unk_8 = bits[25:24]     # (data >> 0x18) & 3
+  unk_9 = bits[27:26]     # (data >> 0x1a) & 3
+  unk_padding = bits[31:28]
+
+class CDNA_PKT_14(PacketType):
+  """pkt_fmt=14: 64-bit packet (case 0x34)"""
+  encoding = bits[3:0] == 14
+  unk_0 = bits[5:5]       # (data >> 5) & 1
+  unk_1 = bits[9:6]       # (data >> 6) & 0xf
+  unk_2 = bits[11:10]     # (data >> 10) & 3
+  unk_3 = bits[24:12]     # (data >> 0xc) & 0x1fff
+  unk_4 = bits[37:25]     # (data >> 0x19) & 0x1fff
+  unk_5 = bits[50:38]     # (data >> 0x26) & 0x1fff
+  unk_6 = bits[51:51]     # (data >> 0x33) & 1
+  unk_padding = bits[63:52]
+
+class CDNA_PKT_15(PacketType):
+  """pkt_fmt=15: 48-bit packet (case 0x38, same as 0x10)"""
+  encoding = bits[3:0] == 15
+  unk_0 = bits[6:5]       # (data >> 5) & 3
+  unk_1 = bits[7:7]       # (data >> 7) + 1 & 1
+  unk_2 = bits[15:9]      # (data >> 9) & 0x7f
+  unk_padding = bits[47:16]
+
+CDNA_PKT_TYPES: dict[int, type[PacketType]] = {
+  0: CDNA_DELTA, 1: CDNA_TIMESTAMP, 2: CDNA_PKT_2, 3: CDNA_WAVESTART, 4: CDNA_PKT_4,
+  5: CDNA_PKT_5, 6: CDNA_WAVEEND, 10: CDNA_EXEC, 11: CDNA_PKT_11, 13: CDNA_INST, 14: CDNA_PKT_14, 15: CDNA_PKT_15,
+}
+
+# Validate CDNA packet definitions
+for pkt_fmt, pkt_cls in CDNA_PKT_TYPES.items():
+  assert pkt_cls.encoding.default == pkt_fmt, f"{pkt_cls.__name__} encoding {pkt_cls.encoding.default} != pkt_fmt {pkt_fmt}"
+  assert CDNA_PKT_SIZES[pkt_fmt] * 2 == pkt_cls._size_nibbles, f"{pkt_cls.__name__} size {pkt_cls._size_nibbles//2} != {CDNA_PKT_SIZES[pkt_fmt]}"
+
+def decode(data: bytes) -> Iterator[PacketType]:
+  """Decode CDNA SQTT blob using 16-bit header format."""
+  pos, time, ts_offset = 0, 0, None
+  while pos + 2 <= len(data):
+    header = int.from_bytes(data[pos:pos+2], 'little')
+    pkt_fmt = header & 0xf
+    pkt_size = CDNA_PKT_SIZES[pkt_fmt]
+    if pos + pkt_size > len(data): break
+
+    raw = int.from_bytes(data[pos:pos+pkt_size], 'little')
+    # pkt_fmt=0 has delta in bits[11:4], accumulate it
+    if pkt_fmt == 0: time += ((raw >> 4) & 0xff) * 4
+    # pkt_fmt=1 with unk_0=0 is absolute timestamp - use it to anchor time
+    if pkt_fmt == 1 and ((raw >> 4) & 0xfff) == 0:
+      abs_ts = raw >> 16
+      if ts_offset is None: ts_offset = abs_ts - time  # first timestamp: save offset
+      else: time = ((abs_ts - ts_offset) & ~3) - 4     # subsequent: compute time, align to 4, subtract 4
+    pkt_cls = CDNA_PKT_TYPES[pkt_fmt]
+    yield pkt_cls.from_raw(raw, time)
+    pos += pkt_size
+
+if __name__ == "__main__":
+  import sys, pickle
+  if len(sys.argv) < 2:
+    print("Usage: python sqtt_cdna.py <pkl_file>")
+    sys.exit(1)
+  with open(sys.argv[1], "rb") as f:
+    data = pickle.load(f)
+  sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
+  for i, event in enumerate(sqtt_events):
+    print(f"\n=== event {i} ===")
+    for pkt in decode(event.blob):
+      print(f"{pkt._time:8}: {pkt}")

extra/assembly/amd/sqttmap.py

@@ -0,0 +1,132 @@
+# maps SQTT trace packets to instructions.
+from dataclasses import dataclass
+from typing import Iterator
+
+from tinygrad.runtime.support.elf import elf_loader
+
+from extra.assembly.amd.sqtt import decode, print_packets, INST, VALUINST, IMMEDIATE, WAVESTART, WAVEEND, InstOp, PacketType, IMMEDIATE_MASK
+from extra.assembly.amd.dsl import Inst
+from extra.assembly.amd import decode_inst
+from extra.assembly.amd.autogen.rdna3.ins import SOPP, s_endpgm
+from extra.assembly.amd.autogen.rdna3.enum import SOPPOp
+
+@dataclass(frozen=True)
+class InstructionInfo:
+  pc: int
+  wave: int
+  inst: Inst
+
+def map_insts(data:bytes, lib:bytes) -> Iterator[tuple[PacketType, InstructionInfo|None]]:
+  """maps SQTT packets to instructions, yields (packet, instruction_info or None)"""
+  # map pcs to insts
+  pc_map:dict[int, Inst] = {}
+  image, sections, _ = elf_loader(lib)
+  text = next((sh for sh in sections if sh.name == ".text"), None)
+  assert text is not None, "no .text section found"
+  text_off, text_size = text.header.sh_addr, text.header.sh_size
+  offset = text_off
+  while offset < text_off + text_size:
+    inst = decode_inst(image[offset:])
+    pc_map[offset-text_off] = inst
+    offset += inst.size()
+
+  wave_pc:dict[int, int] = {}
+  # only processing packets on one [CU, SIMD] unit
+  def simd_select(p) -> bool: return getattr(p, "cu", 0) == 0 and getattr(p, "simd", 0) == 0
+  for p in decode(data):
+    if not simd_select(p): continue
+    if isinstance(p, WAVESTART):
+      assert p.wave not in wave_pc, "only one inflight wave per unit"
+      wave_pc[p.wave] = 0
+      continue
+    if isinstance(p, WAVEEND):
+      pc = wave_pc.pop(p.wave)
+      yield (p, InstructionInfo(pc, p.wave, s_endpgm()))
+      continue
+    # skip OTHER_ instructions, they don't belong to this unit
+    if isinstance(p, INST) and p.op.name.startswith("OTHER_"): continue
+    if isinstance(p, IMMEDIATE_MASK):
+      # immediate mask may yield multiple times per packet
+      for wave in range(16):
+        if p.mask & (1 << wave):
+          inst = pc_map[pc:=wave_pc[wave]]
+          # can this assert be more strict?
+          assert isinstance(inst, SOPP), f"IMMEDIATE_MASK packet must map to SOPP, got {inst}"
+          wave_pc[wave] += inst.size()
+          yield (p, InstructionInfo(pc, wave, inst))
+      continue
+    if isinstance(p, (VALUINST, INST, IMMEDIATE)):
+      inst = pc_map[pc:=wave_pc[p.wave]]
+      # s_delay_alu doesn't get a packet?
+      if isinstance(inst, SOPP) and inst.op in {SOPPOp.S_DELAY_ALU}:
+        wave_pc[p.wave] += inst.size()
+        inst = pc_map[pc:=wave_pc[p.wave]]
+      # identify a branch instruction, only used for asserts
+      is_branch = isinstance(inst, SOPP) and "BRANCH" in inst.op_name
+      if is_branch: assert isinstance(p, INST) and p.op in {InstOp.JUMP_NO, InstOp.JUMP}, f"branch can only be folowed by jump packets, got {p}"
+      # JUMP handling
+      if isinstance(p, INST) and p.op is InstOp.JUMP:
+        assert is_branch, f"JUMP packet must map to a branch instruction, got {inst}"
+        x = inst.simm16 & 0xffff
+        wave_pc[p.wave] += inst.size() + (x - 0x10000 if x & 0x8000 else x)*4
+      else:
+        if is_branch: assert inst.op != SOPPOp.S_BRANCH, f"S_BRANCH must have a JUMP packet, got {p}"
+        wave_pc[p.wave] += inst.size()
+      yield (p, InstructionInfo(pc, p.wave, inst))
+      continue
+    # for all other packets (VMEMEXEC, ALUEXEC, etc.), yield with None
+    yield (p, None)
+
+# test to compare every packet with the rocprof decoder
+
+def test_rocprof_inst_traces_match(sqtt, prg, target):
+  from tinygrad.viz.serve import llvm_disasm
+  from extra.sqtt.roc import decode as roc_decode, InstExec
+  disasm = {addr+prg.base:inst_disasm for addr, inst_disasm in llvm_disasm(target, prg.lib).items()}
+  rctx = roc_decode([sqtt], {prg.name:disasm})
+  rwaves = rctx.inst_execs[(sqtt.kern, sqtt.exec_tag)]
+  rwaves_iter:dict[int, list[Iterator[InstExec]]] = {} # wave unit (0-15) -> list of inst trace iterators for all executions on that unit
+  for w in rwaves: rwaves_iter.setdefault(w.wave_id, []).append(w.unpack_insts())
+  rwaves_base = next(iter(disasm)) # base program counter
+
+  passed_insts = 0
+  for pkt, info in map_insts(sqtt.blob, prg.lib):
+    if DEBUG >= 2: print_packets([pkt])
+    if info is None: continue
+    if DEBUG >= 2: print(f"{' '*29}{info.inst.disasm()}")
+    rocprof_inst = next(rwaves_iter[info.wave][0])
+    ref_pc = rocprof_inst.pc-rwaves_base
+    # always check pc matches
+    assert ref_pc == info.pc, f"pc mismatch {ref_pc}:{disasm[rocprof_inst.pc][0]} != {info.pc}:{info.inst.disasm()}"
+    # special handling for s_endpgm, it marks the wave completion.
+    if info.inst == s_endpgm():
+      completed_wave = list(rwaves_iter[info.wave].pop(0))
+      assert len(completed_wave) == 0, f"incomplete instructions in wave {info.wave}"
+    # otherwise the packet timestamp is time + "stall"
+    else:
+      assert pkt._time == rocprof_inst.time+rocprof_inst.stall
+    passed_insts += 1
+
+  for k,v in rwaves_iter.items():
+    assert len(v) == 0, f"incomplete wave {k}"
+
+  print(f"passed for {passed_insts} instructions across {len(rwaves)} waves scheduled on {len(rwaves_iter)} wave units")
+
+if __name__ == "__main__":
+  import argparse, pickle, pathlib
+  from tinygrad.helpers import temp, DEBUG
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--profile', type=pathlib.Path, metavar="PATH", help='Path to profile (optional file, default: latest profile)',
+                      default=pathlib.Path(temp("profile.pkl", append_user=True)))
+  parser.add_argument('--kernel', type=str, default=None, metavar="NAME", help='Kernel to focus on (optional name, default: all kernels)')
+  args = parser.parse_args()
+  with open(args.profile, "rb") as f:
+    data = pickle.load(f)
+  sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
+  kern_events = {e.name:e for e in data if type(e).__name__ == "ProfileProgramEvent"}
+  target = next((e for e in data if type(e).__name__ == "ProfileDeviceEvent" and e.device.startswith("AMD"))).props["gfx_target_version"]
+  for e in sqtt_events:
+    if args.kernel is not None and args.kernel != e.kern: continue
+    if not e.itrace: continue
+    print(f"==== {e.kern}")
+    test_rocprof_inst_traces_match(e, kern_events[e.kern], target)

extra/assembly/amd/test/bench_emu.py

@@ -6,7 +6,9 @@ from pathlib import Path
 # Set AMD=1 before importing tinygrad
 os.environ["AMD"] = "1"
 
-from extra.assembly.amd.emu import run_asm as python_run_asm, set_valid_mem_ranges, decode_program
+from extra.assembly.amd.emu import run_asm as python_run_asm, decode_program
+from extra.assembly.amd import decode_inst
+from extra.assembly.amd.autogen.rdna3.ins import SOPP, SOPPOp
 
 REMU_PATH = Path(__file__).parents[3] / "remu/target/release/libremu.so"
 if not REMU_PATH.exists():
@@ -64,6 +66,55 @@ def benchmark_emulator(name: str, run_fn, kernel: bytes, global_size, local_size
 
   return sum(times) / len(times)
 
+def profile_instructions(kernel: bytes):
+  """Profile individual instruction compile times."""
+  from extra.assembly.amd.emu import _get_runner, _canonical_runner_cache
+  from tinygrad.helpers import Context
+  _get_runner.cache_clear()
+  _canonical_runner_cache.clear()
+
+  results = []
+  i = 0
+  while i < len(kernel):
+    inst = decode_inst(kernel[i:])
+    if isinstance(inst, SOPP) and inst.op == SOPPOp.S_CODE_END: break
+    inst_bytes = bytes(kernel[i:i + inst.size() + 4])
+    try: inst_str = repr(inst)
+    except Exception: inst_str = f"<{type(inst).__name__}>"
+
+    # Time the full compile (sink + render + compile)
+    start = time.perf_counter()
+    with Context(CCACHE=0):
+      runner, is_new = _get_runner(inst_bytes)
+    compile_time = time.perf_counter() - start
+
+    results.append({
+      'inst_str': inst_str + ('' if is_new else ' [CACHED]'),
+      'compile_ms': compile_time * 1000 if is_new else 0,
+    })
+    i += inst.size()
+
+  return sorted(results, key=lambda x: x['compile_ms'], reverse=True)
+
+def benchmark_python_split(kernel: bytes, global_size, local_size, args_ptr, rsrc2: int, iterations: int = 5):
+  """Benchmark Python emulator with compile and execution times."""
+  from extra.assembly.amd.emu import _get_runner, _canonical_runner_cache
+  from tinygrad.helpers import Context
+  _get_runner.cache_clear()
+  _canonical_runner_cache.clear()
+  decode_program.cache_clear()
+
+  # Measure compile time (decode_program builds sinks, renders, and compiles)
+  compile_start = time.perf_counter()
+  with Context(CCACHE=0):
+    program = decode_program(kernel)
+  compile_time = time.perf_counter() - compile_start
+  n_compiled = len(_canonical_runner_cache)
+
+  # Execution time
+  exec_time = benchmark_emulator("Python", python_run_asm, kernel, global_size, local_size, args_ptr, rsrc2, iterations)
+  return compile_time, exec_time, len(program), n_compiled
+
 def get_tinygrad_kernel(op_name: str) -> tuple[bytes, tuple, tuple, list[int], dict[int, bytes], int] | None:
   """Get a real tinygrad kernel by operation name. Returns (code, global_size, local_size, buf_sizes, buf_data, rsrc2)."""
   try:
@@ -119,14 +170,36 @@ def get_tinygrad_kernel(op_name: str) -> tuple[bytes, tuple, tuple, list[int], d
     print(f"  Error getting kernel: {e}")
     return None
 
-TINYGRAD_TESTS = ["add", "mul", "reduce_sum", "softmax", "exp", "gelu", "matmul_small"]
+TINYGRAD_TESTS = ["add", "mul", "reduce_sum", "softmax", "exp", "sin", "gelu", "matmul_small"]
 
 def main():
   import argparse
   parser = argparse.ArgumentParser(description="Benchmark RDNA3 emulators")
   parser.add_argument("--iterations", type=int, default=3, help="Number of iterations per benchmark")
+  parser.add_argument("--profile", type=str, default=None, help="Profile instructions for a specific kernel (e.g. 'sin')")
+  parser.add_argument("--top", type=int, default=20, help="Number of top instructions to show in profile")
   args = parser.parse_args()
 
+  # Profile mode: show individual instruction timing
+  if args.profile:
+    kernel_info = get_tinygrad_kernel(args.profile)
+    if kernel_info is None:
+      print(f"Failed to get kernel for '{args.profile}'")
+      return
+    kernel = kernel_info[0]
+    print(f"Profiling instructions for '{args.profile}' kernel...")
+    print("=" * 110)
+    results = profile_instructions(kernel)
+    print(f"{'Instruction':<90} {'Compile(ms)':>12}")
+    print("-" * 110)
+    for r in results[:args.top]:
+      inst = r['inst_str'][:87] + "..." if len(r['inst_str']) > 90 else r['inst_str']
+      print(f"{inst:<90} {r['compile_ms']:>12.3f}")
+    print("-" * 110)
+    total = sum(r['compile_ms'] for r in results)
+    print(f"{'TOTAL':<90} {total:>12.3f}")
+    return
+
   rust_remu = get_rust_remu()
   if rust_remu is None:
     print("Rust libremu not found. Build with: cargo build --release --manifest-path extra/remu/Cargo.toml")
@@ -149,44 +222,45 @@ def main():
       continue
 
     kernel, global_size, local_size, buf_sizes, buf_data, rsrc2 = kernel_info
-    n_insts = count_instructions(kernel)
+    buffers, args_arr, args_ptr, ranges = setup_buffers(buf_sizes, buf_data)
+
+    # Benchmark Python emulator (must be first to measure compile time before cache is populated)
+    py_compile, py_exec, n_insts, n_compiled = benchmark_python_split(kernel, global_size, local_size, args_ptr, rsrc2, args.iterations)
+
     n_workgroups = global_size[0] * global_size[1] * global_size[2]
     n_threads = local_size[0] * local_size[1] * local_size[2]
     total_work = n_insts * n_workgroups * n_threads
 
-    print(f"{n_insts} insts × {n_workgroups} WGs × {n_threads} threads = {total_work:,} ops")
-
-    buffers, args_arr, args_ptr, ranges = setup_buffers(buf_sizes, buf_data)
-    set_valid_mem_ranges(ranges)
-
-    py_time = benchmark_emulator("Python", python_run_asm, kernel, global_size, local_size, args_ptr, rsrc2, args.iterations)
+    print(f"{n_insts} insts ({n_compiled} unique) × {n_workgroups} WGs × {n_threads} threads = {total_work:,} ops")
     rust_time = benchmark_emulator("Rust", rust_remu.run_asm, kernel, global_size, local_size, args_ptr, rsrc2, args.iterations) if rust_remu else None
 
-    if py_time:
-      py_rate = total_work / py_time / 1e6
-      print(f"  Python: {py_time*1000:8.3f} ms  ({py_rate:7.2f} M ops/s)")
+    if py_compile is not None:
+      py_exec_rate = total_work / py_exec / 1e6
+      print(f"  Compile:        {py_compile*1000:8.3f} ms  ({n_compiled} unique)")
+      print(f"  Exec:           {py_exec*1000:8.3f} ms  ({py_exec_rate:7.2f} M ops/s)")
     if rust_time:
       rust_rate = total_work / rust_time / 1e6
-      speedup = py_time / rust_time if py_time else 0
-      print(f"  Rust:   {rust_time*1000:8.3f} ms  ({rust_rate:7.2f} M ops/s)  [{speedup:.1f}x faster]")
+      speedup = py_exec / rust_time if py_exec else 0
+      print(f"  Rust:           {rust_time*1000:8.3f} ms  ({rust_rate:7.2f} M ops/s)  [{speedup:.1f}x faster]")
 
-    results.append((op_name, n_insts, n_workgroups, py_time, rust_time))
+    results.append((op_name, n_insts, n_compiled, n_workgroups, py_compile, py_exec, rust_time))
 
   # Summary table
-  print("\n" + "=" * 90)
+  print("\n" + "=" * 110)
   print("SUMMARY")
-  print("=" * 90)
-  print(f"{'Name':<25} {'Insts':<8} {'WGs':<6} {'Python (ms)':<14} {'Rust (ms)':<14} {'Speedup':<10}")
-  print("-" * 90)
+  print("=" * 110)
+  print(f"{'Name':<16} {'Insts':<6} {'Unique':<6} {'WGs':<5} {'Compile (ms)':<14} {'Exec (ms)':<12} {'Rust (ms)':<12} {'Speedup':<10}")
+  print("-" * 110)
 
-  for name, n_insts, n_wgs, py_time, rust_time in results:
-    py_ms = f"{py_time*1000:.3f}" if py_time else "error"
+  for name, n_insts, n_compiled, n_wgs, py_compile, py_exec, rust_time in results:
+    compile_ms = f"{py_compile*1000:.3f}" if py_compile else "error"
+    exec_ms = f"{py_exec*1000:.3f}" if py_exec else "error"
     if rust_time:
       rust_ms = f"{rust_time*1000:.3f}"
-      speedup = f"{py_time/rust_time:.1f}x" if py_time else "N/A"
+      speedup = f"{py_exec/rust_time:.1f}x" if py_exec else "N/A"
     else:
       rust_ms, speedup = "N/A", "N/A"
-    print(f"{name:<25} {n_insts:<8} {n_wgs:<6} {py_ms:<14} {rust_ms:<14} {speedup:<10}")
+    print(f"{name:<16} {n_insts:<6} {n_compiled:<6} {n_wgs:<5} {compile_ms:<14} {exec_ms:<12} {rust_ms:<12} {speedup:<10}")
 
 if __name__ == "__main__":
   main()

extra/assembly/amd/test/external_test_usability.py

@@ -1,196 +0,0 @@
-# Usability tests for the RDNA3 ASM DSL
-# These tests demonstrate how the DSL *should* work for a good user experience
-# Currently many of these tests fail - they document desired behavior
-
-import unittest
-from extra.assembly.amd.autogen.rdna3.ins import *
-from extra.assembly.amd.dsl import Inst, RawImm, SGPR, VGPR
-
-class TestRegisterSliceSyntax(unittest.TestCase):
-  """
-  Issue: Register slice syntax should use AMD assembly convention (inclusive end).
-
-  In AMD assembly, s[4:7] means registers s4, s5, s6, s7 (4 registers, inclusive).
-  The DSL should match this convention so that:
-  - s[4:7] gives 4 registers
-  - Disassembler output can be copied directly back into DSL code
-
-  Fix: Change _RegFactory.__getitem__ to use inclusive end:
-    key.stop - key.start + 1  (instead of key.stop - key.start)
-  """
-  def test_register_slice_count(self):
-    # s[4:7] should give 4 registers: s4, s5, s6, s7 (AMD convention, inclusive)
-    reg = s[4:7]
-    self.assertEqual(reg.count, 4, "s[4:7] should give 4 registers (s4, s5, s6, s7)")
-
-  def test_register_slice_roundtrip(self):
-    # Round-trip: DSL -> disasm -> DSL should preserve register count
-    reg = s[4:7]  # 4 registers in AMD convention
-    inst = s_load_b128(reg, s[0:1], NULL, 0)
-    disasm = inst.disasm()
-    # Disasm shows s[4:7] - user should be able to copy this back
-    self.assertIn("s[4:7]", disasm)
-    # And s[4:7] in DSL should give the same 4 registers
-    reg_from_disasm = s[4:7]
-    self.assertEqual(reg_from_disasm.count, 4, "s[4:7] from disasm should give 4 registers")
-
-
-class TestReprReadability(unittest.TestCase):
-  """
-  Issue: repr() leaks internal RawImm type and omits zero-valued fields.
-
-  When you create v_mov_b32_e32(v[0], v[1]), the repr shows:
-    VOP1(op=1, src0=RawImm(257))
-
-  Problems:
-  1. vdst=v[0] is omitted because 0 is treated as "default"
-  2. src0 shows RawImm(257) instead of v[1]
-  3. User sees encoded values (257 = 256 + 1) instead of register names
-
-  Expected repr: VOP1(op=1, vdst=v[0], src0=v[1])
-  """
-  def test_repr_shows_registers_not_raw_imm(self):
-    inst = v_mov_b32_e32(v[0], v[1])
-    # Should show v[1], not RawImm(257)
-    self.assertNotIn("RawImm", repr(inst), "repr should not expose RawImm internal type")
-    self.assertIn("v[1]", repr(inst), "repr should show register name")
-
-  def test_repr_includes_zero_dst(self):
-    inst = v_mov_b32_e32(v[0], v[1])
-    # v[0] is a valid destination register, should be shown
-    self.assertIn("vdst", repr(inst), "repr should include vdst even when 0")
-
-  def test_repr_roundtrip(self):
-    # repr should produce something that can be eval'd back
-    inst = v_mov_b32_e32(v[0], v[1])
-    # This would require repr to output valid Python, e.g.:
-    # "VOP1(op=VOP1Op.V_MOV_B32, vdst=v[0], src0=v[1])"
-    r = repr(inst)
-    # At minimum, it should be human-readable
-    self.assertIn("v[", r, "repr should show register syntax")
-
-
-class TestInstructionEquality(unittest.TestCase):
-  """
-  Issue: No __eq__ method - instruction comparison requires repr() workaround.
-
-  Two identical instructions should compare equal with ==, but currently:
-    inst1 == inst2 returns False
-
-  The test_handwritten.py works around this with:
-    self.assertEqual(repr(self.inst), repr(reasm))
-  """
-  def test_identical_instructions_equal(self):
-    inst1 = v_mov_b32_e32(v[0], v[1])
-    inst2 = v_mov_b32_e32(v[0], v[1])
-    self.assertEqual(inst1, inst2, "identical instructions should be equal")
-
-  def test_different_instructions_not_equal(self):
-    inst1 = v_mov_b32_e32(v[0], v[1])
-    inst2 = v_mov_b32_e32(v[0], v[2])
-    self.assertNotEqual(inst1, inst2, "different instructions should not be equal")
-
-
-class TestVOPDHelperSignature(unittest.TestCase):
-  """
-  Issue: VOPD helper functions have confusing semantics.
-
-  v_dual_mul_f32 is defined as:
-    v_dual_mul_f32 = functools.partial(VOPD, VOPDOp.V_DUAL_MUL_F32)
-
-  This binds VOPDOp.V_DUAL_MUL_F32 to the FIRST positional arg of VOPD.__init__,
-  which is 'opx'. So v_dual_mul_f32 sets the X operation.
-
-  But then test_dual_mul in test_handwritten.py does:
-    v_dual_mul_f32(VOPDOp.V_DUAL_MUL_F32, vdstx=v[0], ...)
-
-  This passes V_DUAL_MUL_F32 as the SECOND positional arg (opy), making both
-  X and Y operations the same. This is confusing because:
-  1. The function name suggests it handles the X operation
-  2. But you still pass an opcode as the first arg (which becomes opy)
-
-  Expected: Either make the helper fully specify both ops, or make the
-  signature clearer about what the positional arg means.
-  """
-  def test_vopd_helper_opy_should_be_required(self):
-    # Using only keyword args "works" but opy silently defaults to 0
-    inst = v_dual_mul_f32(vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
-    self.assertEqual(inst.opx, VOPDOp.V_DUAL_MUL_F32)
-    # Bug: opy defaults to 0 (V_DUAL_FMAC_F32) silently - should require explicit opy
-    # This test documents the bug - it should fail once fixed
-    self.assertNotEqual(inst.opy, VOPDOp.V_DUAL_FMAC_F32, "opy should not silently default to FMAC")
-
-  def test_vopd_helper_positional_arg_is_opy(self):
-    # The first positional arg after the partial becomes opy, not a second opx
-    inst = v_dual_mul_f32(VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=v[2], vsrcx1=v[3], srcy0=v[4], vsrcy1=v[5])
-    self.assertEqual(inst.opx, VOPDOp.V_DUAL_MUL_F32)  # From partial
-    self.assertEqual(inst.opy, VOPDOp.V_DUAL_MOV_B32)  # From first positional arg
-
-
-class TestFieldAccessPreservesType(unittest.TestCase):
-  """
-  Issue: Field access loses type information.
-
-  After creating an instruction, accessing fields returns encoded int values:
-    inst = v_mov_b32_e32(v[0], v[1])
-    inst.vdst  # returns 0, not VGPR(0)
-
-  This makes it impossible to round-trip register types through field access.
-  """
-  def test_vdst_returns_register(self):
-    inst = v_mov_b32_e32(v[5], v[1])
-    vdst = inst.vdst
-    # Should return a VGPR, not an int
-    self.assertIsInstance(vdst, (VGPR, int), "vdst should return VGPR or at least be usable")
-    # Ideally: self.assertIsInstance(vdst, VGPR)
-
-  def test_src_returns_register_for_vgpr_source(self):
-    inst = v_mov_b32_e32(v[0], v[1])
-    # src0 is encoded as 257 (256 + 1 for v1)
-    # Ideally it should decode back to v[1]
-    src0_raw = inst._values.get('src0')
-    # Currently returns RawImm(257), should return VGPR(1) or similar
-    self.assertNotIsInstance(src0_raw, RawImm, "source should not be RawImm internally")
-
-
-class TestArgumentDiscoverability(unittest.TestCase):
-  """
-  Issue: No clear signature for positional arguments.
-
-  inspect.signature(s_load_b128) shows: (*args, literal=None, **kwargs)
-
-  Users have no way to know the argument order without reading source code.
-  The order is implicitly defined by the class field definition order.
-
-  Possible fixes:
-  1. Add explicit parameter names to functools.partial
-  2. Generate type stubs with proper signatures
-  3. Add docstrings listing the expected arguments
-  """
-  def test_signature_has_named_params(self):
-    import inspect
-    sig = inspect.signature(s_load_b128)
-    params = list(sig.parameters.keys())
-    # Currently: ['args', 'literal', 'kwargs'] (from *args, literal=None, **kwargs)
-    # Expected: something like ['sdata', 'sbase', 'soffset', 'offset', 'literal']
-    self.assertIn('sdata', params, "signature should show field names")
-
-
-class TestSpecialConstants(unittest.TestCase):
-  """
-  Issue: NULL and other constants are IntEnum values that might be confusing.
-
-  NULL = SrcEnum.NULL = 124, but users might expect NULL to be a special object
-  that clearly represents "no register" rather than a magic number.
-  """
-  def test_null_has_clear_repr(self):
-    # NULL should have a clear string representation
-    self.assertIn("NULL", str(NULL) or repr(NULL), "NULL should be clearly identifiable")
-
-  def test_null_is_distinguishable_from_int(self):
-    # NULL should be distinguishable from the raw integer 124
-    self.assertNotEqual(type(NULL), int, "NULL should not be plain int")
-
-
-if __name__ == "__main__":
-  unittest.main()

extra/assembly/amd/test/helpers.py

@@ -5,6 +5,7 @@ from dataclasses import dataclass
 @dataclass
 class KernelInfo:
   code: bytes
+  src: str
   global_size: tuple[int, int, int]
   local_size: tuple[int, int, int]
   buf_idxs: list[int]  # indices into shared buffer pool
@@ -23,44 +24,15 @@ def get_llvm_objdump():
     if shutil.which(p): return p
   raise FileNotFoundError("llvm-objdump not found")
 
-# ═══════════════════════════════════════════════════════════════════════════════
-# EXECUTION CONTEXT (for testing compiled pseudocode)
-# ═══════════════════════════════════════════════════════════════════════════════
+ARCH_TO_TARGET:dict[str, list[str]] = {
+  "rdna3":["gfx1100"],
+  "rdna4":["gfx1200"],
+  "cdna":["gfx950", "gfx942"],
+}
 
-class ExecContext:
-  """Context for running compiled pseudocode in tests."""
-  def __init__(self, s0=0, s1=0, s2=0, d0=0, scc=0, vcc=0, lane=0, exec_mask=0xffffffff, literal=0, vgprs=None, src0_idx=0, vdst_idx=0):
-    from extra.assembly.amd.pcode import Reg, MASK32, MASK64, TypedView
-    self._Reg, self._MASK64, self._TypedView = Reg, MASK64, TypedView
-    self.S0, self.S1, self.S2 = Reg(s0), Reg(s1), Reg(s2)
-    self.D0, self.D1 = Reg(d0), Reg(0)
-    self.SCC, self.VCC, self.EXEC = Reg(scc), Reg(vcc), Reg(exec_mask)
-    self.tmp, self.saveexec = Reg(0), Reg(exec_mask)
-    self.lane, self.laneId, self.literal = lane, lane, literal
-    self.SIMM16, self.SIMM32 = Reg(literal), Reg(literal)
-    self.VGPR = vgprs if vgprs is not None else {}
-    self.SRC0, self.VDST = Reg(src0_idx), Reg(vdst_idx)
+TARGET_TO_ARCH:dict[str, str] = {t:arch for arch,targets in ARCH_TO_TARGET.items() for t in targets}
 
-  def run(self, code: str):
-    """Execute compiled code."""
-    import extra.assembly.amd.pcode as pcode
-    ns = {k: getattr(pcode, k) for k in dir(pcode) if not k.startswith('_')}
-    # Also include underscore-prefixed helpers that compiled pseudocode uses
-    for k in ['_pack', '_pack32']:
-      if hasattr(pcode, k): ns[k] = getattr(pcode, k)
-    ns.update({
-      'S0': self.S0, 'S1': self.S1, 'S2': self.S2, 'D0': self.D0, 'D1': self.D1,
-      'SCC': self.SCC, 'VCC': self.VCC, 'EXEC': self.EXEC,
-      'EXEC_LO': self._TypedView(self.EXEC, 31, 0), 'EXEC_HI': self._TypedView(self.EXEC, 63, 32),
-      'tmp': self.tmp, 'saveexec': self.saveexec,
-      'lane': self.lane, 'laneId': self.laneId, 'literal': self.literal,
-      'SIMM16': self.SIMM16, 'SIMM32': self.SIMM32, 'VGPR': self.VGPR, 'SRC0': self.SRC0, 'VDST': self.VDST,
-    })
-    exec(code, ns)
-    def _sync(ctx_reg, ns_val):
-      if isinstance(ns_val, self._Reg): ctx_reg._val = ns_val._val
-      else: ctx_reg._val = int(ns_val) & self._MASK64
-    for name in ('SCC', 'VCC', 'EXEC', 'D0', 'D1', 'tmp', 'saveexec'):
-      if ns.get(name) is not getattr(self, name): _sync(getattr(self, name), ns[name])
+def get_target(arch:str) -> str: return ARCH_TO_TARGET[arch][0]
 
-  def result(self) -> dict: return {"d0": self.D0._val, "scc": self.SCC._val & 1}
+def get_mattr(arch:str) -> str:
+  return {"rdna3":"+real-true16,+wavefrontsize32", "rdna4":"+real-true16,+wavefrontsize32", "cdna":"+wavefrontsize64"}[arch]

extra/assembly/amd/test/hw/helpers.py

@@ -1,14 +1,25 @@
 """Test infrastructure for hardware-validated RDNA3 emulator tests.
 
 Uses run_asm() with memory output, so tests can run on both emulator and real hardware.
-Set USE_HW=1 to run on both emulator and real hardware, comparing results.
+Set USE_HW=1 to run on both emulator and hardware, comparing results.
 """
-import ctypes, os, struct
+import ctypes, math, os, struct
 from extra.assembly.amd.autogen.rdna3.ins import *
 
-from extra.assembly.amd.emu import WaveState, run_asm, set_valid_mem_ranges
+from extra.assembly.amd.emu import run_asm
 from extra.assembly.amd.dsl import NULL, SCC, VCC_LO, VCC_HI, EXEC_LO, EXEC_HI, M0
-from extra.assembly.amd.pcode import _i32, _f32
+
+def _i32(f: float) -> int: return struct.unpack('<I', struct.pack('<f', f))[0]
+def _f32(i: int) -> float: return struct.unpack('<f', struct.pack('<I', i & 0xFFFFFFFF))[0]
+
+# f16 conversion helpers
+def f16(i: int) -> float: return struct.unpack('<e', struct.pack('<H', i & 0xFFFF))[0]
+def f32_to_f16(f: float) -> int:
+  f = float(f)
+  if math.isnan(f): return 0x7e00
+  if math.isinf(f): return 0x7c00 if f > 0 else 0xfc00
+  try: return struct.unpack('<H', struct.pack('<e', f))[0]
+  except OverflowError: return 0x7c00 if f > 0 else 0xfc00
 
 # For backwards compatibility with tests using SrcEnum.NULL etc.
 class SrcEnum:
@@ -32,11 +43,28 @@ VCC = VCC_LO  # For VOP3SD sdst field (VCC_LO is exported from dsl)
 USE_HW = os.environ.get("USE_HW", "0") == "1"
 FLOAT_TOLERANCE = 1e-5
 
-# Output buffer layout: vgpr[16][32], sgpr[16], vcc, scc
+def get_gpu_target() -> tuple[int, int, int]:
+  """Get the GPU target as (major, minor, stepping) tuple."""
+  if not USE_HW: return (0, 0, 0)
+  from tinygrad.device import Device
+  return Device["AMD"].target
+
+def skip_unless_gfx(min_major: int, min_minor: int = 0, reason: str = ""):
+  """Skip test if GPU target is below the minimum required version."""
+  import unittest
+  def decorator(test_func):
+    if not USE_HW: return test_func
+    target = get_gpu_target()
+    if target[0] < min_major or (target[0] == min_major and target[1] < min_minor):
+      return unittest.skip(reason or f"requires gfx{min_major}{min_minor}0+")(test_func)
+    return test_func
+  return decorator
+
+# Output buffer layout: vgpr[16][32], sgpr[16], vcc, scc, exec
 N_VGPRS, N_SGPRS, WAVE_SIZE = 16, 16, 32
 VGPR_BYTES = N_VGPRS * WAVE_SIZE * 4  # 16 regs * 32 lanes * 4 bytes = 2048
 SGPR_BYTES = N_SGPRS * 4  # 16 regs * 4 bytes = 64
-OUT_BYTES = VGPR_BYTES + SGPR_BYTES + 8  # + vcc + scc
+OUT_BYTES = VGPR_BYTES + SGPR_BYTES + 12  # + vcc + scc + exec
 
 # Float conversion helpers
 def f2i(f: float) -> int: return _i32(f)
@@ -47,6 +75,14 @@ def i642f(i: int) -> float: return struct.unpack('<d', struct.pack('<Q', i))[0]
 def assemble(instructions: list) -> bytes:
   return b''.join(inst.to_bytes() for inst in instructions)
 
+# Simple WaveState class for test output parsing (mirrors emu.py interface for tests)
+class WaveState:
+  def __init__(self):
+    self.vgpr = [[0] * 256 for _ in range(32)]  # vgpr[lane][reg]
+    self.sgpr = [0] * 128
+    self.vcc = 0
+    self.scc = 0
+
 def get_prologue_epilogue(n_lanes: int) -> tuple[list, list]:
   """Generate prologue and epilogue instructions for state capture."""
   prologue = [
@@ -63,6 +99,10 @@ def get_prologue_epilogue(n_lanes: int) -> tuple[list, list]:
   epilogue = [
     s_mov_b32(s[90], VCC_LO),
     s_cselect_b32(s[91], 1, 0),
+    # Save EXEC early (before we modify it for VGPR stores)
+    s_mov_b32(s[95], EXEC_LO),
+    # Restore EXEC to all active lanes for VGPR stores (test may have modified EXEC)
+    s_mov_b32(EXEC_LO, (1 << n_lanes) - 1),
     s_load_b64(s[92:93], s[80:81], 0, soffset=NULL),
     s_waitcnt(0),  # simm16=0 waits for all
     v_lshlrev_b32_e32(v[240], 2, v[255]),
@@ -80,6 +120,9 @@ def get_prologue_epilogue(n_lanes: int) -> tuple[list, list]:
   epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=VGPR_BYTES + SGPR_BYTES))
   epilogue.append(v_mov_b32_e32(v[243], s[91]))
   epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=VGPR_BYTES + SGPR_BYTES + 4))
+  # Store EXEC (saved earlier in s[95])
+  epilogue.append(v_mov_b32_e32(v[243], s[95]))
+  epilogue.append(global_store_b32(addr=v[240], data=v[243], saddr=s[92:93], offset=VGPR_BYTES + SGPR_BYTES + 8))
   epilogue.append(s_mov_b32(EXEC_LO, s[94]))
   epilogue.append(s_endpgm())
   return prologue, epilogue
@@ -95,6 +138,8 @@ def parse_output(out_buf: bytes, n_lanes: int) -> WaveState:
     st.sgpr[i] = struct.unpack_from('<I', out_buf, VGPR_BYTES + i * 4)[0]
   st.vcc = struct.unpack_from('<I', out_buf, VGPR_BYTES + SGPR_BYTES)[0]
   st.scc = struct.unpack_from('<I', out_buf, VGPR_BYTES + SGPR_BYTES + 4)[0]
+  # Store EXEC in its proper location (index 126)
+  st.sgpr[EXEC_LO.offset] = struct.unpack_from('<I', out_buf, VGPR_BYTES + SGPR_BYTES + 8)[0]
   return st
 
 def run_program_emu(instructions: list, n_lanes: int = 1) -> WaveState:
@@ -110,10 +155,10 @@ def run_program_emu(instructions: list, n_lanes: int = 1) -> WaveState:
   kernel_buf = (ctypes.c_char * len(code)).from_buffer_copy(code)
   lib_ptr = ctypes.addressof(kernel_buf)
 
-  set_valid_mem_ranges({(out_addr, OUT_BYTES), (args_ptr, 8)})
   # rsrc2: USER_SGPR_COUNT=2, ENABLE_SGPR_WORKGROUP_ID_X/Y/Z=1, LDS_SIZE=128 (64KB)
   rsrc2 = 0x19c | (128 << 15)
-  result = run_asm(lib_ptr, len(code), 1, 1, 1, n_lanes, 1, 1, args_ptr, rsrc2)
+  scratch_size = 0x10000  # 64KB per lane, matches .amdhsa_private_segment_fixed_size in run_program_hw
+  result = run_asm(lib_ptr, len(code), 1, 1, 1, n_lanes, 1, 1, args_ptr, rsrc2, scratch_size)
   assert result == 0, f"run_asm failed with {result}"
 
   return parse_output(bytes(out_buf), n_lanes)
@@ -148,6 +193,8 @@ test:
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_kernarg_size 8
   .amdhsa_group_segment_fixed_size 65536
+  .amdhsa_private_segment_fixed_size 65536
+  .amdhsa_enable_private_segment 1
 .end_amdhsa_kernel
 
 .amdgpu_metadata
@@ -160,7 +207,7 @@ amdhsa.kernels:
     .symbol: test.kd
     .kernarg_segment_size: 8
     .group_segment_fixed_size: 65536
-    .private_segment_fixed_size: 0
+    .private_segment_fixed_size: 65536
     .kernarg_segment_align: 8
     .wavefront_size: 32
     .sgpr_count: 96
@@ -174,6 +221,7 @@ amdhsa.kernels:
   prg = AMDProgram(dev, "test", lib)
 
   out_gpu = dev.allocator.alloc(OUT_BYTES)
+  assert out_gpu.va_addr % 16 == 0, f"buffer not 16-byte aligned: 0x{out_gpu.va_addr:x}"
   prg(out_gpu, global_size=(1, 1, 1), local_size=(n_lanes, 1, 1), wait=True)
 
   out_buf = bytearray(OUT_BYTES)
@@ -181,8 +229,12 @@ amdhsa.kernels:
 
   return parse_output(bytes(out_buf), n_lanes)
 
-def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgprs: int = N_VGPRS) -> list[str]:
-  """Compare two WaveStates and return list of differences."""
+def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgprs: int = N_VGPRS, ulp_tolerance: int = 0) -> list[str]:
+  """Compare two WaveStates and return list of differences.
+
+  Args:
+    ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
+  """
   import math
   diffs = []
   for i in range(n_vgprs):
@@ -193,6 +245,11 @@ def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgp
         emu_f, hw_f = _f32(emu_val), _f32(hw_val)
         if math.isnan(emu_f) and math.isnan(hw_f):
           continue
+        # Check ULP difference for floats (only for same-sign values)
+        if ulp_tolerance > 0 and (emu_val < 0x80000000) == (hw_val < 0x80000000):
+          ulp_diff = abs(int(emu_val) - int(hw_val))
+          if ulp_diff <= ulp_tolerance:
+            continue
         diffs.append(f"v[{i}] lane {lane}: emu=0x{emu_val:08x} ({emu_f:.6g}) hw=0x{hw_val:08x} ({hw_f:.6g})")
   for i in range(N_SGPRS):
     emu_val = emu_st.sgpr[i]
@@ -205,16 +262,19 @@ def compare_wave_states(emu_st: WaveState, hw_st: WaveState, n_lanes: int, n_vgp
     diffs.append(f"scc: emu={emu_st.scc} hw={hw_st.scc}")
   return diffs
 
-def run_program(instructions: list, n_lanes: int = 1) -> WaveState:
+def run_program(instructions: list, n_lanes: int = 1, ulp_tolerance: int = 0) -> WaveState:
   """Run instructions and return WaveState.
 
   If USE_HW=1, runs on both emulator and hardware, compares results, and raises if they differ.
   Otherwise, runs only on emulator.
+
+  Args:
+    ulp_tolerance: Allow up to this many ULPs difference for float comparisons (0 = exact match required)
   """
   emu_st = run_program_emu(instructions, n_lanes)
   if USE_HW:
     hw_st = run_program_hw(instructions, n_lanes)
-    diffs = compare_wave_states(emu_st, hw_st, n_lanes)
+    diffs = compare_wave_states(emu_st, hw_st, n_lanes, ulp_tolerance=ulp_tolerance)
     if diffs:
       raise AssertionError(f"Emulator vs Hardware mismatch:\n" + "\n".join(diffs))
     return hw_st

extra/assembly/amd/test/hw/test_ds.py

@@ -138,6 +138,100 @@ class TestDS2AddrMore(unittest.TestCase):
     self.assertEqual(st.vgpr[0][4], 0x12345678, "v4 should be untouched")
 
 
+class TestDSB96(unittest.TestCase):
+  """Tests for DS_STORE_B96 and DS_LOAD_B96 (96-bit / 3 dwords)."""
+
+  def test_ds_store_load_b96(self):
+    """DS_STORE_B96 stores 3 VGPRs, DS_LOAD_B96 loads them back."""
+    instructions = [
+      v_mov_b32_e32(v[10], 0),
+      s_mov_b32(s[0], 0x11111111),
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[0], 0x22222222),
+      v_mov_b32_e32(v[1], s[0]),
+      s_mov_b32(s[0], 0x33333333),
+      v_mov_b32_e32(v[2], s[0]),
+      ds_store_b96(addr=v[10], data0=v[0:2]),
+      s_waitcnt(lgkmcnt=0),
+      ds_load_b96(addr=v[10], vdst=v[4:6]),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have first dword")
+    self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have second dword")
+    self.assertEqual(st.vgpr[0][6], 0x33333333, "v6 should have third dword")
+
+  def test_ds_store_b96_with_offset(self):
+    """DS_STORE_B96 with non-zero offset."""
+    instructions = [
+      v_mov_b32_e32(v[10], 0),
+      s_mov_b32(s[0], 0xAAAAAAAA),
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[0], 0xBBBBBBBB),
+      v_mov_b32_e32(v[1], s[0]),
+      s_mov_b32(s[0], 0xCCCCCCCC),
+      v_mov_b32_e32(v[2], s[0]),
+      DS(DSOp.DS_STORE_B96, addr=v[10], data0=v[0:2], offset0=12),
+      s_waitcnt(lgkmcnt=0),
+      DS(DSOp.DS_LOAD_B96, addr=v[10], vdst=v[4:6], offset0=12),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA)
+    self.assertEqual(st.vgpr[0][5], 0xBBBBBBBB)
+    self.assertEqual(st.vgpr[0][6], 0xCCCCCCCC)
+
+
+class TestDSB128(unittest.TestCase):
+  """Tests for DS_STORE_B128 and DS_LOAD_B128 (128-bit / 4 dwords)."""
+
+  def test_ds_store_load_b128(self):
+    """DS_STORE_B128 stores 4 VGPRs, DS_LOAD_B128 loads them back."""
+    instructions = [
+      v_mov_b32_e32(v[10], 0),
+      s_mov_b32(s[0], 0x11111111),
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[0], 0x22222222),
+      v_mov_b32_e32(v[1], s[0]),
+      s_mov_b32(s[0], 0x33333333),
+      v_mov_b32_e32(v[2], s[0]),
+      s_mov_b32(s[0], 0x44444444),
+      v_mov_b32_e32(v[3], s[0]),
+      ds_store_b128(addr=v[10], data0=v[0:3]),
+      s_waitcnt(lgkmcnt=0),
+      ds_load_b128(addr=v[10], vdst=v[4:7]),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][4], 0x11111111, "v4 should have first dword")
+    self.assertEqual(st.vgpr[0][5], 0x22222222, "v5 should have second dword")
+    self.assertEqual(st.vgpr[0][6], 0x33333333, "v6 should have third dword")
+    self.assertEqual(st.vgpr[0][7], 0x44444444, "v7 should have fourth dword")
+
+  def test_ds_store_b128_with_offset(self):
+    """DS_STORE_B128 with non-zero offset."""
+    instructions = [
+      v_mov_b32_e32(v[10], 0),
+      s_mov_b32(s[0], 0xAAAAAAAA),
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[0], 0xBBBBBBBB),
+      v_mov_b32_e32(v[1], s[0]),
+      s_mov_b32(s[0], 0xCCCCCCCC),
+      v_mov_b32_e32(v[2], s[0]),
+      s_mov_b32(s[0], 0xDDDDDDDD),
+      v_mov_b32_e32(v[3], s[0]),
+      DS(DSOp.DS_STORE_B128, addr=v[10], data0=v[0:3], offset0=16),
+      s_waitcnt(lgkmcnt=0),
+      DS(DSOp.DS_LOAD_B128, addr=v[10], vdst=v[4:7], offset0=16),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][4], 0xAAAAAAAA)
+    self.assertEqual(st.vgpr[0][5], 0xBBBBBBBB)
+    self.assertEqual(st.vgpr[0][6], 0xCCCCCCCC)
+    self.assertEqual(st.vgpr[0][7], 0xDDDDDDDD)
+
+
 class TestDSAtomic(unittest.TestCase):
   """Tests for DS atomic operations."""
 
@@ -625,5 +719,47 @@ class TestAtomicOrdering(unittest.TestCase):
     self.assertEqual(st.vgpr[0][4], 150, "Final value should be 150")
 
 
+class TestDsPermute(unittest.TestCase):
+  """Tests for DS_PERMUTE_B32 and DS_BPERMUTE_B32 instructions."""
+
+  def test_ds_permute_b32_identity(self):
+    """DS_PERMUTE_B32 with identity permutation (lane 0 sends to lane 0)."""
+    # For simplicity, test with single lane
+    instructions = [
+      v_mov_b32_e32(v[0], 0),  # addr = 0 (lane 0)
+      v_mov_b32_e32(v[1], 0xDEADBEEF),  # data
+      ds_permute_b32(v[2], v[0], v[1]),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Lane 0 sends to lane 0, so lane 0 gets 0xDEADBEEF
+    self.assertEqual(st.vgpr[0][2], 0xDEADBEEF)
+
+  def test_ds_bpermute_b32_identity(self):
+    """DS_BPERMUTE_B32 with identity permutation (each lane reads from itself)."""
+    instructions = [
+      v_mov_b32_e32(v[0], 0),  # addr = 0 (read from lane 0)
+      v_mov_b32_e32(v[1], 0xCAFEBABE),  # data in lane 0
+      ds_bpermute_b32(v[2], v[0], v[1]),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Lane 0 reads from lane 0's v[1]
+    self.assertEqual(st.vgpr[0][2], 0xCAFEBABE)
+
+  def test_ds_permute_b32_broadcast(self):
+    """DS_PERMUTE_B32 broadcast - all lanes send to lane 0."""
+    # With 4 lanes, all sending to lane 0, highest lane wins
+    instructions = [
+      v_mov_b32_e32(v[0], 0),  # All lanes send to addr 0 (lane 0)
+      v_mov_b32_e32(v[1], 0x11111111),  # All lanes send same data
+      ds_permute_b32(v[2], v[0], v[1]),
+      s_waitcnt(lgkmcnt=0),
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # Lane 0 receives data (highest numbered active lane wins)
+    self.assertEqual(st.vgpr[0][2], 0x11111111)
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_global.py

@@ -128,6 +128,169 @@ class TestGlobalLoad(unittest.TestCase):
 class TestGlobalStore(unittest.TestCase):
   """Tests for GLOBAL store instructions."""
 
+  def test_global_store_b8_basic(self):
+    """GLOBAL_STORE_B8 stores a single byte from VDATA[7:0]."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # First store 0xDEADBEEF to memory
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Now store single byte 0x42 to same address (should only change byte 0)
+      v_mov_b32_e32(v[2], 0x42),
+      global_store_b8(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Read back and check
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Only byte 0 should change from 0xEF to 0x42
+    self.assertEqual(st.vgpr[0][0], 0xDEADBE42, "Only byte 0 should be modified")
+
+  def test_global_store_b8_byte1(self):
+    """GLOBAL_STORE_B8 at offset+1 stores to byte 1."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[2], 0x42),
+      global_store_b8(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+1),
+      s_waitcnt(vmcnt=0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEAD42EF, "Only byte 1 should be modified")
+
+  def test_global_store_b16_basic(self):
+    """GLOBAL_STORE_B16 stores a 16-bit value from VDATA[15:0]."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      s_mov_b32(s[4], 0xCAFE),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEADCAFE, "Only lower 16 bits should be modified")
+
+  def test_global_store_b16_high_half(self):
+    """GLOBAL_STORE_B16 at offset+2 stores to high 16 bits."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      s_mov_b32(s[4], 0xCAFE),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+2),
+      s_waitcnt(vmcnt=0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xCAFEBEEF, "Only upper 16 bits should be modified")
+
+  def test_global_store_b16_byte_offset_1(self):
+    """GLOBAL_STORE_B16 at byte offset 1 stores bytes 1-2 within the same word."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDDCCBBAA),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Store 0xBEEF at byte offset 1 (bytes 1-2)
+      s_mov_b32(s[4], 0xBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+1),
+      s_waitcnt(vmcnt=0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Bytes 1-2 should be 0xBEEF (0xEF at byte 1, 0xBE at byte 2)
+    # Original: 0xDDCCBBAA -> bytes [AA, BB, CC, DD]
+    # After:    0xDDBEEFAA -> bytes [AA, EF, BE, DD]
+    self.assertEqual(st.vgpr[0][0], 0xDDBEEFAA, "Bytes 1-2 should be 0xBEEF")
+
+  def test_global_store_b16_cross_word_boundary(self):
+    """GLOBAL_STORE_B16 at byte offset 3 crosses word boundary (byte 3 of word N, byte 0 of word N+1)."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Initialize two consecutive words
+      s_mov_b32(s[4], 0xDDCCBBAA),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_mov_b32(s[4], 0x44332211),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      # Store 0xBEEF at byte offset 3 (crosses word boundary)
+      # Low byte (0xEF) goes to byte 3 of first word
+      # High byte (0xBE) goes to byte 0 of second word
+      s_mov_b32(s[4], 0xBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b16(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+3),
+      s_waitcnt(vmcnt=0),
+      # Load back both words
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[3], data=v[3], saddr=s[2:3], offset=TEST_OFFSET),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[4], data=v[4], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      v_mov_b32_e32(v[1], v[4]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # First word: 0xDDCCBBAA -> 0xEFCCBBAA (byte 3 becomes 0xEF)
+    # Second word: 0x44332211 -> 0x443322BE (byte 0 becomes 0xBE)
+    self.assertEqual(st.vgpr[0][0], 0xEFCCBBAA, "Byte 3 of first word should be 0xEF")
+    self.assertEqual(st.vgpr[0][1], 0x443322BE, "Byte 0 of second word should be 0xBE")
+
   def test_global_store_b64_basic(self):
     """GLOBAL_STORE_B64 stores 8 bytes from v[n:n+1] to memory."""
     TEST_OFFSET = 256
@@ -360,5 +523,157 @@ class TestD16HiLoads(unittest.TestCase):
     self.assertEqual(byte5, 0x00, f"byte5: expected 0x00, got 0x{byte5:02x}")
 
 
+class TestGlobalOffset(unittest.TestCase):
+  """Tests for GLOBAL instructions with different offsets.
+
+  These tests verify that instruction deduplication correctly handles different offset values.
+  If offset is made dynamic incorrectly, instructions with different offsets may load/store wrong data.
+  """
+
+  def test_global_load_different_offsets(self):
+    """Load from two different offsets and verify correct values."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+      # Store 0xAAAAAAAA at offset 100
+      s_mov_b32(s[0], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=100),
+      # Store 0xBBBBBBBB at offset 200
+      s_mov_b32(s[0], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=200),
+      s_waitcnt(vmcnt=0),
+      # Load from offset 100 -> should get 0xAAAAAAAA
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[3], saddr=SrcEnum.NULL, offset=100),
+      # Load from offset 200 -> should get 0xBBBBBBBB
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[4], saddr=SrcEnum.NULL, offset=200),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      v_mov_b32_e32(v[1], v[4]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
+    self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
+
+  def test_global_store_different_offsets(self):
+    """Store to two different offsets and verify correct values."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+      # Store 0x11111111 at offset 300
+      s_mov_b32(s[0], 0x11111111),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=300),
+      # Store 0x22222222 at offset 400
+      s_mov_b32(s[0], 0x22222222),
+      v_mov_b32_e32(v[3], s[0]),
+      global_store_b32(addr=v[0:1], data=v[3], saddr=SrcEnum.NULL, offset=400),
+      s_waitcnt(vmcnt=0),
+      # Load back to verify
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[4], saddr=SrcEnum.NULL, offset=300),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0:1], vdst=v[5], saddr=SrcEnum.NULL, offset=400),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[4]),
+      v_mov_b32_e32(v[1], v[5]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x11111111, f"offset 300: expected 0x11111111, got 0x{st.vgpr[0][0]:08x}")
+    self.assertEqual(st.vgpr[0][1], 0x22222222, f"offset 400: expected 0x22222222, got 0x{st.vgpr[0][1]:08x}")
+
+  def test_global_negative_offset_no_saddr(self):
+    """Test negative offset without saddr (VGPR pair for address).
+    Store 0xAAAA at offset 100, 0xBBBB at offset 200.
+    Load with offset -100 from vaddr pointing to base+200 -> should get 0xAAAA (at 100).
+    Load with offset -100 from vaddr pointing to base+300 -> should get 0xBBBB (at 200)."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+      # Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
+      s_mov_b32(s[0], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=100),
+      s_mov_b32(s[0], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0:1], data=v[2], saddr=SrcEnum.NULL, offset=200),
+      s_waitcnt(vmcnt=0),
+      # vaddr = base+200, load with offset -100 -> should get value at 100
+      s_add_u32(s[4], s[2], 200),
+      s_addc_u32(s[5], s[3], 0),
+      v_mov_b32_e32(v[4], s[4]),
+      v_mov_b32_e32(v[5], s[5]),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[4:5], vdst=v[6], saddr=SrcEnum.NULL, offset=-100),
+      # vaddr = base+300, load with offset -100 -> should get value at 200
+      s_add_u32(s[4], s[2], 300),
+      s_addc_u32(s[5], s[3], 0),
+      v_mov_b32_e32(v[4], s[4]),
+      v_mov_b32_e32(v[5], s[5]),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[4:5], vdst=v[7], saddr=SrcEnum.NULL, offset=-100),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[6]),
+      v_mov_b32_e32(v[1], v[7]),
+      v_mov_b32_e32(v[4], 0),
+      v_mov_b32_e32(v[5], 0),
+      v_mov_b32_e32(v[6], 0),
+      v_mov_b32_e32(v[7], 0),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+      s_mov_b32(s[4], 0),
+      s_mov_b32(s[5], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
+    self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
+
+  def test_global_negative_offset_with_saddr(self):
+    """Test negative offset with saddr (SGPR pair for base address).
+    Store 0xAAAA at offset 100, 0xBBBB at offset 200.
+    Load with offset -100 from saddr pointing to base+200 -> should get 0xAAAA (at 100).
+    Load with offset -100 from saddr pointing to base+300 -> should get 0xBBBB (at 200)."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
+      s_mov_b32(s[0], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
+      s_mov_b32(s[0], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[0]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
+      s_waitcnt(vmcnt=0),
+      # saddr = base+200, load with offset -100 -> should get value at 100
+      s_add_u32(s[4], s[2], 200),
+      s_addc_u32(s[5], s[3], 0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[6], saddr=s[4:5], offset=-100),
+      # saddr = base+300, load with offset -100 -> should get value at 200
+      s_add_u32(s[4], s[2], 300),
+      s_addc_u32(s[5], s[3], 0),
+      GLOBAL(GLOBALOp.GLOBAL_LOAD_B32, addr=v[0], vdst=v[7], saddr=s[4:5], offset=-100),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[6]),
+      v_mov_b32_e32(v[1], v[7]),
+      v_mov_b32_e32(v[6], 0),
+      v_mov_b32_e32(v[7], 0),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+      s_mov_b32(s[4], 0),
+      s_mov_b32(s[5], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.vgpr[0][0]:08x}")
+    self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.vgpr[0][1]:08x}")
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_scratch.py

@@ -0,0 +1,355 @@
+"""Tests for SCRATCH instructions - scratch (private) memory operations.
+
+Includes: scratch_load_*, scratch_store_*
+"""
+import unittest
+from extra.assembly.amd.test.hw.helpers import *
+
+class TestScratchStore(unittest.TestCase):
+  """Tests for SCRATCH store instructions."""
+
+  def test_scratch_store_b32_basic(self):
+    """SCRATCH_STORE_B32 stores 32-bit value to scratch memory."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      # Store via scratch
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Load back via scratch
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
+
+  def test_scratch_store_b64_basic(self):
+    """SCRATCH_STORE_B64 stores 64-bit value to scratch memory."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      s_mov_b32(s[5], 0xCAFEBABE),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[3], s[5]),
+      v_mov_b32_e32(v[0], 0),
+      scratch_store_b64(addr=v[0], data=v[2:3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_b64(addr=v[0], vdst=v[4:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[4]),
+      v_mov_b32_e32(v[1], v[5]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
+    self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
+
+  def test_scratch_store_b8_basic(self):
+    """SCRATCH_STORE_B8 stores single byte to scratch memory."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # First store full word
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Store single byte
+      v_mov_b32_e32(v[2], 0x42),
+      scratch_store_b8(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Load back
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Only byte 0 should change from 0xEF to 0x42
+    self.assertEqual(st.vgpr[0][0], 0xDEADBE42)
+
+  def test_scratch_store_b16_basic(self):
+    """SCRATCH_STORE_B16 stores 16-bit value to scratch memory."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[0], 0),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      s_mov_b32(s[4], 0xCAFE),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b16(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEADCAFE)
+
+
+class TestScratchLoad(unittest.TestCase):
+  """Tests for SCRATCH load instructions."""
+
+  def test_scratch_load_b96(self):
+    """SCRATCH_LOAD_B96 loads 96-bit value correctly."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[4]),
+      s_mov_b32(s[4], 0xBBBBBBBB),
+      v_mov_b32_e32(v[3], s[4]),
+      s_mov_b32(s[4], 0xCCCCCCCC),
+      v_mov_b32_e32(v[4], s[4]),
+      scratch_store_b96(addr=v[0], data=v[2:4], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_b96(addr=v[0], vdst=v[5:7], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[5]),
+      v_mov_b32_e32(v[1], v[6]),
+      v_mov_b32_e32(v[2], v[7]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA)
+    self.assertEqual(st.vgpr[0][1], 0xBBBBBBBB)
+    self.assertEqual(st.vgpr[0][2], 0xCCCCCCCC)
+
+  def test_scratch_load_b128(self):
+    """SCRATCH_LOAD_B128 loads 128-bit value correctly."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[4]),
+      s_mov_b32(s[4], 0xCAFEBABE),
+      v_mov_b32_e32(v[3], s[4]),
+      s_mov_b32(s[4], 0x12345678),
+      v_mov_b32_e32(v[4], s[4]),
+      s_mov_b32(s[4], 0x9ABCDEF0),
+      v_mov_b32_e32(v[5], s[4]),
+      scratch_store_b128(addr=v[0], data=v[2:5], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_b128(addr=v[0], vdst=v[6:9], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[6]),
+      v_mov_b32_e32(v[1], v[7]),
+      v_mov_b32_e32(v[2], v[8]),
+      v_mov_b32_e32(v[3], v[9]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xDEADBEEF)
+    self.assertEqual(st.vgpr[0][1], 0xCAFEBABE)
+    self.assertEqual(st.vgpr[0][2], 0x12345678)
+    self.assertEqual(st.vgpr[0][3], 0x9ABCDEF0)
+
+  def test_scratch_load_u8(self):
+    """SCRATCH_LOAD_U8 loads unsigned byte with zero extension."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xDEADBEAB),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_u8(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xAB)
+
+  def test_scratch_load_i8(self):
+    """SCRATCH_LOAD_I8 loads signed byte with sign extension."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0x80),  # -128 as signed byte
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b8(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_i8(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFF80)
+
+  def test_scratch_load_u16(self):
+    """SCRATCH_LOAD_U16 loads unsigned 16-bit with zero extension."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xDEADCAFE),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_u16(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xCAFE)
+
+  def test_scratch_load_i16(self):
+    """SCRATCH_LOAD_I16 loads signed 16-bit with sign extension."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0x8000),  # -32768 as signed 16-bit
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b16(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      scratch_load_i16(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFF8000)
+
+
+class TestScratchSVE(unittest.TestCase):
+  """Tests for SCRATCH SVE (Scratch VGPR Enable) bit behavior."""
+
+  def test_scratch_sve_zero_ignores_vaddr(self):
+    """With SVE=0, VADDR should be ignored in address calculation."""
+    TEST_OFFSET = 256
+    # Store a marker value at offset 256 (where SVE=0 should go)
+    # Then set v[0] to a non-zero value (100) and store via scratch with SVE=0
+    # If SVE=0 is handled correctly, the VADDR (100) should be IGNORED,
+    # and the store should go to offset 256, not 256+100=356
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # First, store 0xAAAAAAAA at offset 256 with v[0]=0
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
+      s_waitcnt(vmcnt=0),
+      # Now set v[0] to 100 (non-zero) and store 0xBBBBBBBB with SVE=0
+      # With SVE=0, v[0] should be IGNORED, so this should overwrite offset 256
+      v_mov_b32_e32(v[0], 100),
+      s_mov_b32(s[4], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
+      s_waitcnt(vmcnt=0),
+      # Load back from offset 256 (with v[0]=0) - should get 0xBBBBBBBB
+      v_mov_b32_e32(v[0], 0),
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=0),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # If SVE=0 works correctly, v[0] should be 0xBBBBBBBB (the second store overwrote the first)
+    # If SVE=0 is wrong (VADDR used), v[0] would be 0xAAAAAAAA (stores went to different locations)
+    self.assertEqual(st.vgpr[0][0], 0xBBBBBBBB, "SVE=0 should ignore VADDR, both stores should go to same location")
+
+  def test_scratch_sve_one_uses_vaddr(self):
+    """With SVE=1, VADDR should be used as offset in address calculation."""
+    TEST_OFFSET = 256
+    # Store at offset 256 with v[0]=0, then store at offset 256 with v[0]=100 and SVE=1
+    # With SVE=1, the second store should go to 256+100=356, not 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # First, store 0xAAAAAAAA at offset 256 with v[0]=0
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
+      s_waitcnt(vmcnt=0),
+      # Now set v[0] to 100 and store 0xBBBBBBBB with SVE=1
+      # With SVE=1, v[0] IS used, so this should go to offset 256+100=356
+      v_mov_b32_e32(v[0], 100),
+      s_mov_b32(s[4], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[4]),
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
+      s_waitcnt(vmcnt=0),
+      # Load back from offset 256 (with v[0]=0) - should still be 0xAAAAAAAA
+      v_mov_b32_e32(v[0], 0),
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET, sve=1),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # If SVE=1 works correctly, v[0] should be 0xAAAAAAAA (stores went to different locations)
+    self.assertEqual(st.vgpr[0][0], 0xAAAAAAAA, "SVE=1 should use VADDR, stores should go to different locations")
+
+
+class TestScratchMultiLane(unittest.TestCase):
+  """Tests for SCRATCH operations with multiple lanes."""
+
+  def test_scratch_store_load_multi_lane(self):
+    """SCRATCH store/load works correctly with multiple lanes (private per-lane memory)."""
+    TEST_OFFSET = 256
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=SrcEnum.NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Each lane stores its lane ID
+      v_mov_b32_e32(v[0], 0),
+      v_mov_b32_e32(v[2], v[255]),  # v[255] has packed workitem IDs, low 10 bits = x
+      v_and_b32_e32(v[2], 0x3FF, v[2]),  # extract lane ID
+      scratch_store_b32(addr=v[0], data=v[2], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      # Load back
+      scratch_load_b32(addr=v[0], vdst=v[3], saddr=SrcEnum.NULL, offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      v_mov_b32_e32(v[0], v[3]),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # Each lane should have loaded its own lane ID
+    for lane in range(4):
+      self.assertEqual(st.vgpr[lane][0], lane, f"Lane {lane} should have value {lane}")
+
+
+if __name__ == '__main__':
+  unittest.main()

extra/assembly/amd/test/hw/test_smem.py

@@ -0,0 +1,448 @@
+"""Tests for SMEM instructions - scalar memory operations.
+
+Includes: s_load_b32, s_load_b64, s_load_b128, s_load_b256, s_load_b512
+Tests both immediate and register offset addressing modes.
+"""
+import unittest
+from extra.assembly.amd.test.hw.helpers import *
+
+# Use offset into output buffer for test data (output buffer is 2124 bytes)
+TEST_OFFSET = 2000
+
+# Cache invalidation sequence for scalar loads after vector stores
+# s_wait_idle waits for all outstanding memory operations including cache flushes
+CACHE_INV = [s_gl1_inv(), s_dcache_inv(), s_wait_idle()]
+
+class TestSLoadRegisterOffset(unittest.TestCase):
+  """Tests for s_load with register offset (soffset field).
+
+  Bug: s_load_b32(s[dst], s[base:base+1], s[off]) ignores the register offset
+  and only uses the immediate offset field. This causes incorrect memory loads
+  when the offset comes from a register.
+  """
+
+  def test_s_load_b32_register_offset_basic(self):
+    """s_load_b32 with register offset should load from base + reg_offset."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values to output buffer: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      s_mov_b32(s[5], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[3], s[5]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Now test s_load with register offset
+      # Put offset value in s[4]: offset = 4 bytes (1 dword)
+      s_mov_b32(s[4], 4),
+      # Load from out_ptr + TEST_OFFSET + s[4] (should load 0xBBBBBBBB)
+      s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[5], 0xBBBBBBBB,
+      f"s_load with reg offset 4 should load 0xBBBBBBBB: s[5]=0x{st.sgpr[5]:08x}")
+
+  def test_s_load_b32_register_offset_different_from_immediate(self):
+    """s_load_b32 with register offset loads different data than immediate offset 0."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      s_mov_b32(s[5], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[4]),
+      v_mov_b32_e32(v[3], s[5]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load with immediate offset 0
+      s_load_b32(s[5], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Load with register offset 4
+      s_mov_b32(s[4], 4),
+      s_load_b32(s[6], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # s[5] has dword at offset 0 (0xAAAAAAAA), s[6] has dword at offset 4 (0xBBBBBBBB)
+    self.assertEqual(st.sgpr[5], 0xAAAAAAAA)
+    self.assertEqual(st.sgpr[6], 0xBBBBBBBB)
+    self.assertNotEqual(st.sgpr[5], st.sgpr[6],
+      f"s_load with reg offset 4 should load different value than offset 0: "
+      f"s[5]=0x{st.sgpr[5]:08x}, s[6]=0x{st.sgpr[6]:08x}")
+
+  def test_s_load_b32_register_offset_same_as_dst(self):
+    """s_load_b32 where soffset register is same as destination.
+
+    This is the exact pattern that exposes the bug:
+    s_load_b32(s[8], s[2:3], s[8])
+    The offset should be read BEFORE the destination is overwritten.
+    """
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
+      s_mov_b32(s[6], 0xAAAAAAAA),
+      s_mov_b32(s[7], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[6]),
+      v_mov_b32_e32(v[3], s[7]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Set up s[4] = 4 (offset in bytes)
+      s_mov_b32(s[4], 4),
+      # Load using s[4] as both offset and destination
+      # Should load from base + 4, then store result in s[4]
+      s_load_b32(s[4], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Also load with immediate offset 4 for comparison
+      s_load_b32(s[5], s[2:3], NULL, offset=TEST_OFFSET+4),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # s[4] and s[5] should have the same value (both loaded from offset 4 = 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[4], 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[4], st.sgpr[5],
+      f"s_load with reg offset s[4]=4 should match immediate offset=4: "
+      f"s[4]=0x{st.sgpr[4]:08x}, s[5]=0x{st.sgpr[5]:08x}")
+
+  def test_s_load_b32_register_offset_zero(self):
+    """s_load_b32 with register offset = 0 should be same as immediate offset 0."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test value: 0xDEADBEEF at offset
+      s_mov_b32(s[7], 0xDEADBEEF),
+      v_mov_b32_e32(v[2], s[7]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load with register offset 0
+      s_mov_b32(s[4], 0),
+      s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Load with immediate offset 0
+      s_load_b32(s[6], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[5], 0xDEADBEEF)
+    self.assertEqual(st.sgpr[5], st.sgpr[6],
+      f"s_load with reg offset 0 should match immediate offset 0: "
+      f"s[5]=0x{st.sgpr[5]:08x}, s[6]=0x{st.sgpr[6]:08x}")
+
+  def test_s_load_b32_register_plus_immediate_offset(self):
+    """s_load_b32 with both register and immediate offset should add them."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values: 0xAAAAAAAA at offset, 0xBBBBBBBB at offset+4
+      s_mov_b32(s[8], 0xAAAAAAAA),
+      s_mov_b32(s[9], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[8]),
+      v_mov_b32_e32(v[3], s[9]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # reg offset = 4, imm offset = 0 -> total offset = 4
+      s_mov_b32(s[4], 4),
+      s_load_b32(s[5], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # reg offset = 0, imm offset = 4 -> total offset = 4
+      s_mov_b32(s[6], 0),
+      s_load_b32(s[7], s[2:3], s[6], offset=TEST_OFFSET+4),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Both should load from offset 4 (0xBBBBBBBB)
+    self.assertEqual(st.sgpr[5], 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[7], 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[5], st.sgpr[7],
+      f"reg_off=4 + imm_off=0 should equal reg_off=0 + imm_off=4: "
+      f"s[5]=0x{st.sgpr[5]:08x}, s[7]=0x{st.sgpr[7]:08x}")
+
+
+class TestSLoadMultiDword(unittest.TestCase):
+  """Tests for multi-dword s_load with register offset."""
+
+  def test_s_load_b64_register_offset(self):
+    """s_load_b64 with register offset should load 2 dwords from base + reg_offset."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values: 0xAAAAAAAA, 0xBBBBBBBB at offset
+      s_mov_b32(s[10], 0xAAAAAAAA),
+      s_mov_b32(s[11], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[10]),
+      v_mov_b32_e32(v[3], s[11]),
+      v_mov_b32_e32(v[0], 0),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      global_store_b32(addr=v[0], data=v[3], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load with register offset 0
+      s_mov_b32(s[4], 0),
+      s_load_b64(s[6:7], s[2:3], s[4], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Compare with immediate offset
+      s_load_b64(s[8:9], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[6], 0xAAAAAAAA)
+    self.assertEqual(st.sgpr[7], 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[6], st.sgpr[8])
+    self.assertEqual(st.sgpr[7], st.sgpr[9])
+
+  def test_s_load_b128_register_offset(self):
+    """s_load_b128 with register offset should load 4 dwords from base + reg_offset."""
+    instructions = [
+      # Load output buffer pointer from args
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      # Store test values: 0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD at offset
+      v_mov_b32_e32(v[0], 0),
+      s_mov_b32(s[14], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[14]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_mov_b32(s[14], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[14]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_mov_b32(s[14], 0xCCCCCCCC),
+      v_mov_b32_e32(v[2], s[14]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+8),
+      s_mov_b32(s[14], 0xDDDDDDDD),
+      v_mov_b32_e32(v[2], s[14]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+12),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load with register offset 0 (s_load_b128 requires 4-aligned dest: s[4], s[8], s[12], ...)
+      s_mov_b32(s[15], 0),
+      s_load_b128(s[4:7], s[2:3], s[15], offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Compare with immediate offset
+      s_load_b128(s[8:11], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(0),
+      # Zero out pointer regs (different addresses in emu vs hw)
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0xAAAAAAAA)
+    self.assertEqual(st.sgpr[5], 0xBBBBBBBB)
+    self.assertEqual(st.sgpr[6], 0xCCCCCCCC)
+    self.assertEqual(st.sgpr[7], 0xDDDDDDDD)
+    self.assertEqual(st.sgpr[4], st.sgpr[8])
+    self.assertEqual(st.sgpr[5], st.sgpr[9])
+
+
+class TestSLoadLarge(unittest.TestCase):
+  """Tests for large s_load operations (s_load_b256, s_load_b512)."""
+
+  def test_s_load_b256_basic(self):
+    """s_load_b256 loads 8 consecutive dwords."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store 8 test values
+      s_mov_b32(s[20], 0x11111111),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET),
+      s_mov_b32(s[20], 0x22222222),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+4),
+      s_mov_b32(s[20], 0x33333333),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+8),
+      s_mov_b32(s[20], 0x44444444),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+12),
+      s_mov_b32(s[20], 0x55555555),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+16),
+      s_mov_b32(s[20], 0x66666666),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+20),
+      s_mov_b32(s[20], 0x77777777),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+24),
+      s_mov_b32(s[20], 0x88888888),
+      v_mov_b32_e32(v[2], s[20]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET+28),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load all 8 dwords with s_load_b256
+      s_load_b256(s[4:11], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0x11111111)
+    self.assertEqual(st.sgpr[5], 0x22222222)
+    self.assertEqual(st.sgpr[6], 0x33333333)
+    self.assertEqual(st.sgpr[7], 0x44444444)
+    self.assertEqual(st.sgpr[8], 0x55555555)
+    self.assertEqual(st.sgpr[9], 0x66666666)
+    self.assertEqual(st.sgpr[10], 0x77777777)
+    self.assertEqual(st.sgpr[11], 0x88888888)
+
+  def test_s_load_b512_basic(self):
+    """s_load_b512 loads 16 consecutive dwords."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store 16 test values (use a pattern: 0x10, 0x20, ..., 0x100)
+      *[instr for i in range(16) for instr in [
+        s_mov_b32(s[20], (i + 1) * 0x11111111),
+        v_mov_b32_e32(v[2], s[20]),
+        global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET + i * 4),
+      ]],
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load all 16 dwords with s_load_b512
+      s_load_b512(s[64:79], s[2:3], NULL, offset=TEST_OFFSET),
+      s_waitcnt(lgkmcnt=0),
+      # Copy results to lower regs for verification (since st.sgpr only has 16 regs in test)
+      s_mov_b32(s[4], s[64]),
+      s_mov_b32(s[5], s[65]),
+      s_mov_b32(s[6], s[78]),
+      s_mov_b32(s[7], s[79]),
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0x11111111, "first dword")
+    self.assertEqual(st.sgpr[5], 0x22222222, "second dword")
+    self.assertEqual(st.sgpr[6], 0xFFFFFFFF & (15 * 0x11111111), "15th dword")
+    self.assertEqual(st.sgpr[7], 0xFFFFFFFF & (16 * 0x11111111), "16th dword")
+
+  def test_s_load_b256_with_register_offset(self):
+    """s_load_b256 with register offset should add reg offset to address."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store pattern at TEST_OFFSET+8: skip first 2 dwords
+      *[instr for i in range(8) for instr in [
+        s_mov_b32(s[20], (i + 1) * 0x11111111),
+        v_mov_b32_e32(v[2], s[20]),
+        global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=TEST_OFFSET + 8 + i * 4),
+      ]],
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load with register offset 8
+      s_mov_b32(s[20], 8),
+      s_load_b256(s[4:11], s[2:3], s[20], offset=TEST_OFFSET),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0x11111111, "first dword at offset+8")
+    self.assertEqual(st.sgpr[5], 0x22222222, "second dword at offset+8")
+    self.assertEqual(st.sgpr[11], 0x88888888, "last dword at offset+8")
+
+
+class TestSLoadOffset(unittest.TestCase):
+  """Tests for s_load with different immediate offsets.
+
+  These tests verify that instruction deduplication correctly handles different offset values.
+  If offset is made dynamic incorrectly, instructions with different offsets may load wrong data.
+  """
+
+  def test_s_load_different_offsets(self):
+    """Load from two different offsets and verify correct values."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store 0xAAAAAAAA at offset 100
+      s_mov_b32(s[4], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
+      # Store 0xBBBBBBBB at offset 200
+      s_mov_b32(s[4], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[4]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # Load from offset 100 -> should get 0xAAAAAAAA
+      s_load_b32(s[4], s[2:3], NULL, offset=100),
+      # Load from offset 200 -> should get 0xBBBBBBBB
+      s_load_b32(s[5], s[2:3], NULL, offset=200),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[2], 0), s_mov_b32(s[3], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0xAAAAAAAA, f"offset 100: expected 0xAAAAAAAA, got 0x{st.sgpr[4]:08x}")
+    self.assertEqual(st.sgpr[5], 0xBBBBBBBB, f"offset 200: expected 0xBBBBBBBB, got 0x{st.sgpr[5]:08x}")
+
+  def test_s_load_negative_offset(self):
+    """Test negative offset (21-bit signed).
+    Store 0xAAAA at offset 100, 0xBBBB at offset 200.
+    Load with offset -100 from base+200 -> should get 0xAAAA.
+    Load with offset -100 from base+300 -> should get 0xBBBB."""
+    instructions = [
+      s_load_b64(s[2:3], s[80:81], 0, soffset=NULL),
+      s_waitcnt(lgkmcnt=0),
+      v_mov_b32_e32(v[0], 0),
+      # Store 0xAAAAAAAA at offset 100, 0xBBBBBBBB at offset 200
+      s_mov_b32(s[8], 0xAAAAAAAA),
+      v_mov_b32_e32(v[2], s[8]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=100),
+      s_mov_b32(s[8], 0xBBBBBBBB),
+      v_mov_b32_e32(v[2], s[8]),
+      global_store_b32(addr=v[0], data=v[2], saddr=s[2:3], offset=200),
+      s_waitcnt(vmcnt=0),
+      *CACHE_INV,
+      # base+200, load with offset -100 -> should get value at 100
+      s_add_u32(s[6], s[2], 200),
+      s_addc_u32(s[7], s[3], 0),
+      s_load_b32(s[4], s[6:7], NULL, offset=-100),
+      # base+300, load with offset -100 -> should get value at 200
+      s_add_u32(s[6], s[2], 300),
+      s_addc_u32(s[7], s[3], 0),
+      s_load_b32(s[5], s[6:7], NULL, offset=-100),
+      s_waitcnt(lgkmcnt=0),
+      s_mov_b32(s[2], 0),
+      s_mov_b32(s[3], 0),
+      s_mov_b32(s[6], 0),
+      s_mov_b32(s[7], 0),
+      s_mov_b32(s[8], 0),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[4], 0xAAAAAAAA, f"offset 200-100=100: expected 0xAAAAAAAA, got 0x{st.sgpr[4]:08x}")
+    self.assertEqual(st.sgpr[5], 0xBBBBBBBB, f"offset 300-100=200: expected 0xBBBBBBBB, got 0x{st.sgpr[5]:08x}")
+
+
+if __name__ == '__main__':
+  unittest.main()

extra/assembly/amd/test/hw/test_sop.py

@@ -62,6 +62,30 @@ class TestBasicScalar(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     self.assertEqual(st.sgpr[1], 0x80000000)
 
+  @skip_unless_gfx(11, 5, "SALU FP ops require gfx1150+")
+  def test_s_fmamk_f32(self):
+    """S_FMAMK_F32: D = S0 * literal + S1."""
+    # 2.0 * 3.0 + 1.0 = 7.0
+    instructions = [
+      s_mov_b32(s[0], f2i(2.0)),
+      s_mov_b32(s[1], f2i(1.0)),
+      s_fmamk_f32(s[2], s[0], s[1], literal=f2i(3.0)),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], f2i(7.0))
+
+  @skip_unless_gfx(11, 5, "SALU FP ops require gfx1150+")
+  def test_s_fmamk_f32_negative(self):
+    """S_FMAMK_F32 with negative values."""
+    # -2.0 * 4.0 + 10.0 = 2.0
+    instructions = [
+      s_mov_b32(s[0], f2i(-2.0)),
+      s_mov_b32(s[1], f2i(10.0)),
+      s_fmamk_f32(s[2], s[0], s[1], literal=f2i(4.0)),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], f2i(2.0))
+
 
 class TestQuadmaskWqm(unittest.TestCase):
   """Tests for S_QUADMASK_B32 and S_WQM_B32."""
@@ -298,6 +322,56 @@ class TestSignedArithmetic(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     self.assertEqual(st.sgpr[2], 2)
 
+  def test_s_mul_hi_u32_max(self):
+    """S_MUL_HI_U32: 0xFFFFFFFF * 0xFFFFFFFF."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),
+      s_mov_b32(s[1], 0xFFFFFFFF),
+      s_mul_hi_u32(s[2], s[0], s[1]),  # (0xFFFFFFFF * 0xFFFFFFFF) >> 32 = 0xFFFFFFFE
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 0xFFFFFFFE)
+
+  def test_s_mul_hi_i32_positive(self):
+    """S_MUL_HI_I32: positive * positive."""
+    instructions = [
+      s_mov_b32(s[0], 0x40000000),  # 2^30
+      s_mov_b32(s[1], 4),
+      s_mul_hi_i32(s[2], s[0], s[1]),  # (2^30 * 4) >> 32 = 1
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 1)
+
+  def test_s_mul_hi_i32_neg_times_neg(self):
+    """S_MUL_HI_I32: (-1) * (-1) = 1, high bits = 0."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # -1
+      s_mov_b32(s[1], 0xFFFFFFFF),  # -1
+      s_mul_hi_i32(s[2], s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 0)
+
+  def test_s_mul_hi_i32_neg_times_pos(self):
+    """S_MUL_HI_I32: (-1) * 2 = -2, high bits = -1 (sign extension)."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # -1
+      s_mov_b32(s[1], 2),
+      s_mul_hi_i32(s[2], s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 0xFFFFFFFF)  # -1 sign extends
+
+  def test_s_mul_hi_i32_min_int(self):
+    """S_MUL_HI_I32: MIN_INT * 2 = -2^32, high = -1."""
+    instructions = [
+      s_mov_b32(s[0], 0x80000000),  # -2^31 (MIN_INT)
+      s_mov_b32(s[1], 2),
+      s_mul_hi_i32(s[2], s[0], s[1]),  # (-2^31 * 2) >> 32 = -1
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 0xFFFFFFFF)
+
   def test_s_mul_i32(self):
     """S_MUL_I32: signed multiply low 32 bits."""
     instructions = [
@@ -329,6 +403,176 @@ class TestSignedArithmetic(unittest.TestCase):
     self.assertEqual(st.sgpr[7], ((dividend * 2) + 1) & 0xFFFFFFFF)
 
 
+class TestBitSet(unittest.TestCase):
+  """Tests for S_BITSET0_B32 and S_BITSET1_B32 instructions."""
+
+  def test_s_bitset1_b32_set_bit0(self):
+    """S_BITSET1_B32: set bit 0 in destination."""
+    instructions = [
+      s_mov_b32(s[0], 0),     # start with 0
+      s_mov_b32(s[1], 0),     # bit position = 0
+      s_bitset1_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 1, "Bit 0 should be set")
+
+  def test_s_bitset1_b32_set_bit31(self):
+    """S_BITSET1_B32: set bit 31 in destination."""
+    instructions = [
+      s_mov_b32(s[0], 0),     # start with 0
+      s_mov_b32(s[1], 31),    # bit position = 31
+      s_bitset1_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0x80000000, "Bit 31 should be set")
+
+  def test_s_bitset1_b32_preserves_other_bits(self):
+    """S_BITSET1_B32: preserves bits not being set."""
+    instructions = [
+      s_mov_b32(s[0], 0xFF00FF00),  # existing pattern
+      s_mov_b32(s[1], 0),            # bit position = 0
+      s_bitset1_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0xFF00FF01, "Should set bit 0 while preserving others")
+
+  def test_s_bitset0_b32_clear_bit0(self):
+    """S_BITSET0_B32: clear bit 0 in destination."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # start with all bits set
+      s_mov_b32(s[1], 0),            # bit position = 0
+      s_bitset0_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0xFFFFFFFE, "Bit 0 should be cleared")
+
+  def test_s_bitset0_b32_clear_bit31(self):
+    """S_BITSET0_B32: clear bit 31 in destination."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # start with all bits set
+      s_mov_b32(s[1], 31),           # bit position = 31
+      s_bitset0_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0x7FFFFFFF, "Bit 31 should be cleared")
+
+  def test_s_bitset1_b32_uses_low5_bits(self):
+    """S_BITSET1_B32: only uses low 5 bits of position (mod 32)."""
+    instructions = [
+      s_mov_b32(s[0], 0),
+      s_mov_b32(s[1], 32 + 5),   # position = 37, but mod 32 = 5
+      s_bitset1_b32(s[0], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0x20, "Bit 5 should be set (37 mod 32 = 5)")
+
+
+class TestBfeI64(unittest.TestCase):
+  """Tests for S_BFE_I64 - 64-bit bit field extract with sign extension.
+
+  Regression tests for sign extension bug where 32-bit masks were incorrectly
+  used for 64-bit operations, causing the high 32 bits to not be sign-extended.
+  """
+
+  def test_s_bfe_i64_positive_no_sign_extend(self):
+    """S_BFE_I64: positive value (1) in 16 bits should not sign extend."""
+    # S1 encodes: [22:16] = width, [5:0] = offset
+    # width=16, offset=0 -> S1 = (16 << 16) | 0 = 0x100000
+    instructions = [
+      s_mov_b32(s[0], 1),         # S0 lo = 1
+      s_mov_b32(s[1], 0),         # S0 hi = 0
+      s_mov_b32(s[2], 0x100000),  # width=16, offset=0
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 1, "lo should be 1")
+    self.assertEqual(st.vgpr[0][1], 0, "hi should be 0 (no sign extend)")
+
+  def test_s_bfe_i64_negative_sign_extend(self):
+    """S_BFE_I64: 0xFFFF (-1 in 16 bits) should sign extend to 64 bits.
+
+    This is the main regression test - before the fix, hi was 0 instead of 0xFFFFFFFF.
+    """
+    instructions = [
+      s_mov_b32(s[0], 0xFFFF),    # S0 lo = -1 in 16 bits
+      s_mov_b32(s[1], 0),         # S0 hi = 0
+      s_mov_b32(s[2], 0x100000),  # width=16, offset=0
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF, "lo should be 0xFFFFFFFF")
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF, "hi should be 0xFFFFFFFF (sign extended)")
+
+  def test_s_bfe_i64_8bit_negative_sign_extend(self):
+    """S_BFE_I64: 0xFF (-1 in 8 bits) should sign extend to 64 bits."""
+    # width=8, offset=0 -> S1 = (8 << 16) | 0 = 0x80000
+    instructions = [
+      s_mov_b32(s[0], 0xFF),      # S0 lo = -1 in 8 bits
+      s_mov_b32(s[1], 0),         # S0 hi = 0
+      s_mov_b32(s[2], 0x80000),   # width=8, offset=0
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF, "lo should be 0xFFFFFFFF")
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF, "hi should be 0xFFFFFFFF (sign extended)")
+
+  def test_s_bfe_i64_8bit_positive(self):
+    """S_BFE_I64: 0x7F (127 in 8 bits) should not sign extend."""
+    # width=8, offset=0 -> S1 = (8 << 16) | 0 = 0x80000
+    instructions = [
+      s_mov_b32(s[0], 0x7F),      # S0 lo = 127 in 8 bits (MSB=0)
+      s_mov_b32(s[1], 0),         # S0 hi = 0
+      s_mov_b32(s[2], 0x80000),   # width=8, offset=0
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x7F, "lo should be 0x7F")
+    self.assertEqual(st.vgpr[0][1], 0, "hi should be 0 (no sign extend)")
+
+  def test_s_bfe_i64_with_offset(self):
+    """S_BFE_I64: extract from non-zero bit offset with sign extension."""
+    # Extract 16 bits starting at bit 8: value 0xFF00 >> 8 = 0xFF = -1 in 8 bits? No wait...
+    # Let's put 0x8000FF00: extract 16 bits at offset 8 = 0x00FF (positive)
+    # Put 0xFF00_0000: extract 16 bits at offset 16 = 0xFF00 = -256 in signed 16-bit
+    instructions = [
+      s_mov_b32(s[0], 0xFF000000),  # bits [31:24] = 0xFF, [23:16] = 0x00
+      s_mov_b32(s[1], 0),
+      # width=16, offset=16 -> S1 = (16 << 16) | 16 = 0x100010
+      s_mov_b32(s[2], 0x100010),
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Extract bits [31:16] = 0xFF00, sign bit is bit 15 of extracted = bit 31 of original = 1
+    # So result should be sign-extended 0xFF00 -> 0xFFFFFF00 in lo, 0xFFFFFFFF in hi
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFF00, "lo should be sign-extended 0xFF00")
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF, "hi should be 0xFFFFFFFF (sign extended)")
+
+  def test_s_bfe_i64_32bit_negative(self):
+    """S_BFE_I64: extract 32 bits with sign extension."""
+    # width=32, offset=0 -> S1 = (32 << 16) | 0 = 0x200000
+    instructions = [
+      s_mov_b32(s[0], 0x80000000),  # MIN_INT32 = -2^31
+      s_mov_b32(s[1], 0),
+      s_mov_b32(s[2], 0x200000),    # width=32, offset=0
+      s_bfe_i64(s[4:5], s[0:1], s[2]),
+      v_mov_b32_e32(v[0], s[4]),
+      v_mov_b32_e32(v[1], s[5]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x80000000, "lo should be 0x80000000")
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF, "hi should be 0xFFFFFFFF (sign extended)")
+
+
 class Test64BitCompare(unittest.TestCase):
   """Tests for 64-bit scalar compare instructions."""
 
@@ -377,5 +621,272 @@ class Test64BitCompare(unittest.TestCase):
     self.assertEqual(st.sgpr[4], 1)
 
 
+class TestSOPPNop(unittest.TestCase):
+  """Tests for S_NOP and other SOPP instructions with expression-based for loops.
+
+  S_NOP's pcode uses 'for i in 0U : SIMM16.u16[3 : 0].u32 do' which requires
+  the parser to handle non-constant loop bounds.
+  """
+
+  def test_s_nop_basic(self):
+    """S_NOP executes without side effects."""
+    # S_NOP with immediate 0 should just do nothing
+    instructions = [
+      s_mov_b32(s[0], 42),
+      s_nop(0),  # nop with simm16=0
+      s_mov_b32(s[1], 100),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 42)
+    self.assertEqual(st.sgpr[1], 100)
+
+  def test_s_nop_with_count(self):
+    """S_NOP with count parameter executes multiple nops."""
+    # S_NOP with immediate 3 should execute 4 nops (0:3 inclusive)
+    instructions = [
+      s_mov_b32(s[0], 1),
+      s_nop(3),  # nop with simm16=3 -> 4 iterations
+      s_add_u32(s[0], s[0], 1),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 2)
+
+
+class TestNullRegister(unittest.TestCase):
+  """Tests for NULL register (124) behavior - writes should be discarded, reads return 0."""
+
+  def test_s_mov_b32_from_null(self):
+    """S_MOV_B32 from NULL should read as 0."""
+    instructions = [
+      s_mov_b32(s[0], 0xDEADBEEF),  # Set s[0] to sentinel
+      s_mov_b32(s[0], NULL),  # Read from NULL - should be 0
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0)
+
+  def test_s_add_u32_with_null_src(self):
+    """S_ADD_U32 with NULL as source should use 0."""
+    instructions = [
+      s_mov_b32(s[0], 100),
+      s_add_u32(s[1], s[0], NULL),  # 100 + 0 = 100
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[1], 100)
+
+  def test_s_mov_b32_to_null(self):
+    """S_MOV_B32 to NULL (sdst=124) should discard the write."""
+    instructions = [
+      s_mov_b32(s[0], 0xDEADBEEF),  # Set s[0] to sentinel
+      s_mov_b32(NULL, 42),  # Write to NULL - should be discarded
+      # s[0] should still be 0xDEADBEEF since NULL write doesn't affect it
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[0], 0xDEADBEEF)
+
+  def test_s_add_u32_to_null(self):
+    """S_ADD_U32 with sdst=NULL should discard result but still set SCC."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),
+      s_mov_b32(s[1], 1),
+      s_add_u32(NULL, s[0], s[1]),  # overflow, write to NULL
+      s_cselect_b32(s[2], 1, 0),  # capture SCC
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # SCC should still be set from overflow even though result was discarded
+    self.assertEqual(st.sgpr[2], 1)
+    self.assertEqual(st.scc, 1)
+
+  def test_s_and_b32_to_null(self):
+    """S_AND_B32 with sdst=NULL should discard result but still set SCC."""
+    instructions = [
+      s_mov_b32(s[0], 0xFF00FF00),
+      s_mov_b32(s[1], 0x0F0F0F0F),
+      s_and_b32(NULL, s[0], s[1]),  # result=0x0F000F00, non-zero so SCC=1
+      s_cselect_b32(s[2], 1, 0),  # capture SCC
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 1)  # SCC=1 because result was non-zero
+    self.assertEqual(st.scc, 1)
+
+  def test_s_or_b32_to_null_zero_result(self):
+    """S_OR_B32 with sdst=NULL and zero result should set SCC=0."""
+    instructions = [
+      s_mov_b32(s[0], 0),
+      s_mov_b32(s[1], 0),
+      s_or_b32(NULL, s[0], s[1]),  # result=0, so SCC=0
+      s_cselect_b32(s[2], 1, 0),  # capture SCC
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.sgpr[2], 0)  # SCC=0 because result was zero
+    self.assertEqual(st.scc, 0)
+
+
+class Test64BitSOP1InlineConstants(unittest.TestCase):
+  """Tests for 64-bit SOP1 instructions with inline constants.
+
+  Regression tests for bug where rsrc_dyn didn't properly handle 64-bit
+  inline constants, incorrectly duplicating lo bits to hi instead of
+  zero/sign-extending.
+  """
+
+  def test_s_mov_b64_inline_0(self):
+    """S_MOV_B64 with inline constant 0."""
+    instructions = [
+      s_mov_b64(s[0:1], 0),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_mov_b64_inline_16(self):
+    """S_MOV_B64 with inline constant 16 should set lo=16, hi=0."""
+    instructions = [
+      s_mov_b64(s[0:1], 16),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 16)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_mov_b64_inline_64(self):
+    """S_MOV_B64 with inline constant 64 (max positive)."""
+    instructions = [
+      s_mov_b64(s[0:1], 64),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 64)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_mov_b64_inline_neg1(self):
+    """S_MOV_B64 with inline constant -1 should sign-extend."""
+    instructions = [
+      s_mov_b64(s[0:1], -1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF)
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF)
+
+  def test_s_mov_b64_inline_neg16(self):
+    """S_MOV_B64 with inline constant -16 should sign-extend."""
+    instructions = [
+      s_mov_b64(s[0:1], -16),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFF0)
+    self.assertEqual(st.vgpr[0][1], 0xFFFFFFFF)
+
+  def test_s_mov_b64_float_const_1_0(self):
+    """S_MOV_B64 with float inline constant 1.0 - casts F32 to F64."""
+    instructions = [
+      s_mov_b64(s[0:1], 1.0),  # inline constant 242 (1.0f)
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Hardware casts F32 to F64: 1.0f64 = 0x3FF0000000000000
+    self.assertEqual(st.vgpr[0][0], 0x00000000)  # lo
+    self.assertEqual(st.vgpr[0][1], 0x3FF00000)  # hi
+
+  def test_s_or_b64_inline_constant(self):
+    """S_OR_B64 with 64-bit inline constant."""
+    instructions = [
+      s_mov_b64(s[0:1], 0),
+      s_or_b64(s[2:3], s[0:1], 16),
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 16)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_and_b64_inline_constant(self):
+    """S_AND_B64 with 64-bit inline constant."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),
+      s_mov_b32(s[1], 0xFFFFFFFF),
+      s_and_b64(s[2:3], s[0:1], 16),
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 16)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+
+class Test64BitSOPLiterals(unittest.TestCase):
+  """Tests for 64-bit SOP instructions with 32-bit literals.
+
+  Tests the behavior when a 64-bit SOP instruction uses a 32-bit literal
+  (offset 255 in instruction encoding). The literal is zero-extended to 64 bits.
+  """
+
+  def test_s_mov_b64_literal(self):
+    """S_MOV_B64 with 32-bit literal value - zero-extended to 64 bits."""
+    instructions = [
+      s_mov_b64(s[0:1], 0x12345678),  # literal > 64, uses literal encoding
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x12345678)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_or_b64_literal(self):
+    """S_OR_B64 with 32-bit literal value - zero-extended to 64 bits."""
+    instructions = [
+      s_mov_b64(s[0:1], 0),
+      s_or_b64(s[2:3], s[0:1], 0x12345678),  # literal
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x12345678)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_and_b64_literal(self):
+    """S_AND_B64 with 32-bit literal value - zero-extended to 64 bits."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),
+      s_mov_b32(s[1], 0xFFFFFFFF),
+      s_and_b64(s[2:3], s[0:1], 0x12345678),  # literal
+      v_mov_b32_e32(v[0], s[2]),
+      v_mov_b32_e32(v[1], s[3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x12345678)
+    self.assertEqual(st.vgpr[0][1], 0)
+
+  def test_s_mov_b64_literal_negative(self):
+    """S_MOV_B64 with 0xFFFFFFFF literal - zero-extended (not sign-extended)."""
+    instructions = [
+      s_mov_b64(s[0:1], 0xFFFFFFFF),  # -1 as 32-bit, but zero-extended to 64-bit
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF)
+    self.assertEqual(st.vgpr[0][1], 0)  # zero-extended, not sign-extended
+
+  def test_s_mov_b64_literal_high_bit(self):
+    """S_MOV_B64 with 0x80000000 literal - zero-extended (not sign-extended)."""
+    instructions = [
+      s_mov_b64(s[0:1], 0x80000000),  # high bit set, but zero-extended
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0x80000000)
+    self.assertEqual(st.vgpr[0][1], 0)  # zero-extended, not sign-extended
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vop1.py

@@ -255,7 +255,6 @@ class TestF16Conversions(unittest.TestCase):
 
   def test_v_cvt_f16_f32_small(self):
     """V_CVT_F16_F32 converts small f32 value."""
-    from extra.assembly.amd.pcode import f32_to_f16
     instructions = [
       v_mov_b32_e32(v[0], 0.5),
       v_cvt_f16_f32_e32(v[1], v[0]),
@@ -293,7 +292,6 @@ class TestF16Conversions(unittest.TestCase):
 
   def test_v_cvt_f16_f32_reads_full_32bit_source(self):
     """V_CVT_F16_F32 must read full 32-bit f32 source."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x3fc00000),  # f32 1.5
       v_mov_b32_e32(v[0], s[0]),
@@ -302,7 +300,7 @@ class TestF16Conversions(unittest.TestCase):
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
     lo_bits = result & 0xffff
-    self.assertEqual(lo_bits, 0x3e00, f"Expected f16(1.5)=0x3e00, got 0x{lo_bits:04x} ({_f16(lo_bits)})")
+    self.assertEqual(lo_bits, 0x3e00, f"Expected f16(1.5)=0x3e00, got 0x{lo_bits:04x} ({f16(lo_bits)})")
 
   def test_v_cvt_i16_f16_zero(self):
     """V_CVT_I16_F16 converts f16 zero to i16 zero."""
@@ -348,6 +346,142 @@ class TestF16Conversions(unittest.TestCase):
     self.assertEqual(result, 1, f"Expected 1 from high bits, got {result}")
 
 
+class TestF64Conversions(unittest.TestCase):
+  """Tests for f64 conversion instructions. Regression tests for f32_to_f64/f64_to_f32."""
+
+  def test_v_cvt_f64_f32_one(self):
+    """V_CVT_F64_F32 converts f32 1.0 to f64."""
+    instructions = [
+      s_mov_b32(s[0], f2i(1.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_f32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, 1.0, places=10)
+
+  def test_v_cvt_f64_f32_negative(self):
+    """V_CVT_F64_F32 converts f32 -2.5 to f64."""
+    instructions = [
+      s_mov_b32(s[0], f2i(-2.5)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_f32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, -2.5, places=10)
+
+  def test_v_cvt_f64_f32_pi(self):
+    """V_CVT_F64_F32 converts f32 pi to f64."""
+    import math
+    instructions = [
+      s_mov_b32(s[0], f2i(3.14159265)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_f32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, 3.14159265, places=5)
+
+  def test_v_cvt_f64_f32_zero(self):
+    """V_CVT_F64_F32 converts f32 0.0 to f64."""
+    instructions = [
+      v_mov_b32_e32(v[0], 0),
+      v_cvt_f64_f32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertEqual(result, 0.0)
+
+  def test_v_cvt_f32_f64_one(self):
+    """V_CVT_F32_F64 converts f64 1.0 to f32."""
+    f64_bits = f2i64(1.0)
+    lo, hi = f64_bits & 0xFFFFFFFF, (f64_bits >> 32) & 0xFFFFFFFF
+    instructions = [
+      s_mov_b32(s[0], lo),
+      s_mov_b32(s[1], hi),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_cvt_f32_f64_e32(v[2], v[0:1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][2])
+    self.assertAlmostEqual(result, 1.0, places=5)
+
+  def test_v_cvt_f32_f64_negative(self):
+    """V_CVT_F32_F64 converts f64 -3.5 to f32."""
+    f64_bits = f2i64(-3.5)
+    lo, hi = f64_bits & 0xFFFFFFFF, (f64_bits >> 32) & 0xFFFFFFFF
+    instructions = [
+      s_mov_b32(s[0], lo),
+      s_mov_b32(s[1], hi),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_cvt_f32_f64_e32(v[2], v[0:1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][2])
+    self.assertAlmostEqual(result, -3.5, places=5)
+
+  def test_v_cvt_f32_f64_large(self):
+    """V_CVT_F32_F64 converts large f64 to f32."""
+    f64_bits = f2i64(123456.789)
+    lo, hi = f64_bits & 0xFFFFFFFF, (f64_bits >> 32) & 0xFFFFFFFF
+    instructions = [
+      s_mov_b32(s[0], lo),
+      s_mov_b32(s[1], hi),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_cvt_f32_f64_e32(v[2], v[0:1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][2])
+    self.assertAlmostEqual(result, 123456.789, places=0)
+
+  def test_v_cvt_f64_i32_positive(self):
+    """V_CVT_F64_I32 converts positive i32 to f64."""
+    instructions = [
+      s_mov_b32(s[0], 42),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_i32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, 42.0, places=10)
+
+  def test_v_cvt_f64_i32_negative(self):
+    """V_CVT_F64_I32 converts negative i32 to f64."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # -1 as i32
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_i32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, -1.0, places=10)
+
+  def test_v_cvt_f64_u32_large(self):
+    """V_CVT_F64_U32 converts large u32 to f64."""
+    instructions = [
+      s_mov_b32(s[0], 0xFFFFFFFF),  # max u32
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_f64_u32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertAlmostEqual(result, 4294967295.0, places=0)
+
+  def test_v_cvt_f64_u32_zero(self):
+    """V_CVT_F64_U32 converts 0 to f64."""
+    instructions = [
+      v_mov_b32_e32(v[0], 0),
+      v_cvt_f64_u32_e32(v[2:3], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i642f((st.vgpr[0][3] << 32) | st.vgpr[0][2])
+    self.assertEqual(result, 0.0)
+
+
 class TestClz(unittest.TestCase):
   """Tests for V_CLZ_I32_U32 - count leading zeros."""
 
@@ -560,7 +694,6 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_abs_negative(self):
     """V_CVT_F32_F16 with |abs| on negative value."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
     instructions = [
       s_mov_b32(s[0], f16_neg1),
@@ -573,7 +706,6 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_abs_positive(self):
     """V_CVT_F32_F16 with |abs| on positive value (should stay positive)."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_2 = f32_to_f16(2.0)  # 0x4000
     instructions = [
       s_mov_b32(s[0], f16_2),
@@ -586,7 +718,6 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_neg_positive(self):
     """V_CVT_F32_F16 with neg on positive value."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_2 = f32_to_f16(2.0)  # 0x4000
     instructions = [
       s_mov_b32(s[0], f16_2),
@@ -599,7 +730,6 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f32_f16_neg_negative(self):
     """V_CVT_F32_F16 with neg on negative value (double negative)."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_neg2 = f32_to_f16(-2.0)  # 0xc000
     instructions = [
       s_mov_b32(s[0], f16_neg2),
@@ -612,7 +742,6 @@ class TestCvtF16Modifiers(unittest.TestCase):
 
   def test_v_cvt_f16_f32_then_pack_for_wmma(self):
     """CVT F32->F16 followed by pack (common WMMA pattern)."""
-    from extra.assembly.amd.pcode import _f16
     f32_val = 3.5
     instructions = [
       s_mov_b32(s[0], f2i(f32_val)),
@@ -621,8 +750,8 @@ class TestCvtF16Modifiers(unittest.TestCase):
       v_pack_b32_f16(v[2], v[1], v[1]),  # Pack same value
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = _f16(st.vgpr[0][2] & 0xffff)
-    hi = _f16((st.vgpr[0][2] >> 16) & 0xffff)
+    lo = f16(st.vgpr[0][2] & 0xffff)
+    hi = f16((st.vgpr[0][2] >> 16) & 0xffff)
     self.assertAlmostEqual(lo, f32_val, places=1)
     self.assertAlmostEqual(hi, f32_val, places=1)
 
@@ -668,7 +797,6 @@ class TestConversionRounding(unittest.TestCase):
 
   def test_f16_to_f32_precision(self):
     """F16 to F32 conversion precision."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_val = f32_to_f16(1.5)
     instructions = [
       s_mov_b32(s[0], f16_val),
@@ -680,7 +808,6 @@ class TestConversionRounding(unittest.TestCase):
 
   def test_f16_denormal_to_f32(self):
     """F16 denormal converts to small positive f32."""
-    from extra.assembly.amd.pcode import _f16
     f16_denorm = 0x0001  # Smallest positive f16 denormal
     instructions = [
       v_mov_b32_e32(v[0], f16_denorm),
@@ -1238,5 +1365,220 @@ class TestFloorEdgeCases(unittest.TestCase):
     self.assertAlmostEqual(i2f(st.vgpr[0][1]), -1.0, places=5)
 
 
+class TestVop1F16HiHalf(unittest.TestCase):
+  """Regression tests for VOP1 f16 hi-half source operand handling.
+
+  For 16-bit VOP1 operations, when src0 is in the range v[128]+ (offset >= 384),
+  the hardware reads from the high 16 bits of v[src0-128]. The emulator must
+  extract bits [31:16] from the actual VGPR.
+  """
+
+  def test_v_cvt_f32_f16_src_hi_half(self):
+    """V_CVT_F32_F16 with source from hi-half (v[128]+).
+
+    When src0 >= v[128], it reads from the high 16 bits of v[src0-128].
+    This is critical for global_load_d16_hi_b16 + v_cvt_f32_f16 patterns.
+
+    Regression test for: VOP1 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v_cvt_f32_f16 v[1], v[128] (reads hi half of v[0])
+      # Should convert f16(2.0) to f32(2.0)
+      v_cvt_f32_f16_e32(v[1], v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][1])
+    self.assertAlmostEqual(result, 2.0, places=5, msg=f"Expected f32(2.0), got {result}")
+
+  def test_v_cvt_f32_f16_src_lo_vs_hi(self):
+    """V_CVT_F32_F16 comparing lo and hi half reads.
+
+    v[0] has different values in lo and hi halves.
+    v_cvt_f32_f16 v[1], v[0] should read lo (1.0)
+    v_cvt_f32_f16 v[2], v[128] should read hi (2.0)
+
+    Regression test for: VOP1 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # Read from lo half
+      v_cvt_f32_f16_e32(v[1], v[0]),
+      # Read from hi half
+      v_cvt_f32_f16_e32(v[2], v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result_lo = i2f(st.vgpr[0][1])
+    result_hi = i2f(st.vgpr[0][2])
+    self.assertAlmostEqual(result_lo, 1.0, places=5, msg=f"Expected f32(1.0) from lo, got {result_lo}")
+    self.assertAlmostEqual(result_hi, 2.0, places=5, msg=f"Expected f32(2.0) from hi, got {result_hi}")
+
+  def test_v_cvt_i16_f16_src_hi_half(self):
+    """V_CVT_I16_F16 with source from hi-half.
+
+    Regression test for: VOP1 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0xc000_3c00: hi=f16(-2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0xc0003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v_cvt_i16_f16 v[1], v[128] (reads hi half of v[0])
+      # Should convert f16(-2.0) to i16(-2)
+      v_cvt_i16_f16_e32(v[1], v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    expected = (-2) & 0xffff
+    self.assertEqual(result, expected, f"Expected i16(-2)=0x{expected:04x}, got 0x{result:04x}")
+
+  def test_v_mov_b16_src_hi_half(self):
+    """V_MOV_B16 with source from hi-half.
+
+    Regression test for: VOP1 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0xBEEF_DEAD: hi=0xBEEF, lo=0xDEAD
+      s_mov_b32(s[0], 0xBEEFDEAD),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = 0x0000_0000 initially
+      v_mov_b32_e32(v[1], 0),
+      # v_mov_b16 v[1], v[128] (reads hi half of v[0])
+      # Should move 0xBEEF to v[1].lo
+      v_mov_b16_e32(v[1], v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertEqual(result, 0xBEEF, f"Expected 0xBEEF from hi half, got 0x{result:04x}")
+
+
+class TestReciprocalF16(unittest.TestCase):
+  """Tests for V_RCP_F16 - reciprocal in half precision.
+
+  The pcode uses a 16-bit float literal: D0.f16 = 16'1.0 / S0.f16
+  This tests that the sized float literal (16'1.0) is correctly parsed.
+  """
+
+  def test_v_rcp_f16_one(self):
+    """V_RCP_F16: 1/1.0 = 1.0"""
+    import struct
+    def f16_to_bits(f): return struct.unpack('<H', struct.pack('<e', f))[0]
+    def bits_to_f16(b): return struct.unpack('<e', struct.pack('<H', b))[0]
+    instructions = [
+      # Load f16 1.0 into low 16 bits of v[0]
+      v_mov_b32_e32(v[0], f16_to_bits(1.0)),
+      v_rcp_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = bits_to_f16(st.vgpr[0][1] & 0xFFFF)
+    self.assertAlmostEqual(result, 1.0, places=2, msg="1/1.0 should be 1.0")
+
+  def test_v_rcp_f16_two(self):
+    """V_RCP_F16: 1/2.0 = 0.5"""
+    import struct
+    def f16_to_bits(f): return struct.unpack('<H', struct.pack('<e', f))[0]
+    def bits_to_f16(b): return struct.unpack('<e', struct.pack('<H', b))[0]
+    instructions = [
+      v_mov_b32_e32(v[0], f16_to_bits(2.0)),
+      v_rcp_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = bits_to_f16(st.vgpr[0][1] & 0xFFFF)
+    self.assertAlmostEqual(result, 0.5, places=2, msg="1/2.0 should be 0.5")
+
+  def test_v_rcp_f16_four(self):
+    """V_RCP_F16: 1/4.0 = 0.25"""
+    import struct
+    def f16_to_bits(f): return struct.unpack('<H', struct.pack('<e', f))[0]
+    def bits_to_f16(b): return struct.unpack('<e', struct.pack('<H', b))[0]
+    instructions = [
+      v_mov_b32_e32(v[0], f16_to_bits(4.0)),
+      v_rcp_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = bits_to_f16(st.vgpr[0][1] & 0xFFFF)
+    self.assertAlmostEqual(result, 0.25, places=2, msg="1/4.0 should be 0.25")
+
+
+class TestCvtNormF16(unittest.TestCase):
+  """Tests for V_CVT_NORM_I16_F16 and V_CVT_NORM_U16_F16."""
+
+  def test_cvt_norm_i16_f16_positive(self):
+    """V_CVT_NORM_I16_F16: f16 1.0 -> i16 max (32767)."""
+    instructions = [
+      s_mov_b32(s[0], f32_to_f16(1.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_norm_i16_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertEqual(result, 32767)
+
+  def test_cvt_norm_i16_f16_negative(self):
+    """V_CVT_NORM_I16_F16: f16 -1.0 -> i16 -32767 (0x8001)."""
+    instructions = [
+      s_mov_b32(s[0], f32_to_f16(-1.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_norm_i16_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertEqual(result, 0x8001)  # -32767, hardware uses symmetric range
+
+  def test_cvt_norm_i16_f16_zero(self):
+    """V_CVT_NORM_I16_F16: f16 0.0 -> i16 0."""
+    instructions = [
+      v_mov_b32_e32(v[0], 0),
+      v_cvt_norm_i16_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertEqual(result, 0)
+
+  def test_cvt_norm_u16_f16_one(self):
+    """V_CVT_NORM_U16_F16: f16 1.0 -> u16 max (65535)."""
+    instructions = [
+      s_mov_b32(s[0], f32_to_f16(1.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_norm_u16_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertEqual(result, 65535)
+
+  def test_cvt_norm_u16_f16_half(self):
+    """V_CVT_NORM_U16_F16: f16 0.5 -> u16 ~32768."""
+    instructions = [
+      s_mov_b32(s[0], f32_to_f16(0.5)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_cvt_norm_u16_f16_e32(v[1], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    self.assertAlmostEqual(result, 32768, delta=1)
+
+
+class TestPermlane64(unittest.TestCase):
+  """Tests for V_PERMLANE64_B32 instruction (wave64 cross-half swap)."""
+
+  def test_v_permlane64_b32_is_nop_in_wave32(self):
+    """V_PERMLANE64_B32 is a NOP in wave32 mode.
+
+    Per AMD pcode: "if WAVE32 then s_nop(...) else ... endif"
+    The emulator runs in wave32 mode, so this instruction should not modify registers.
+    """
+    instructions = [
+      v_mov_b32_e32(v[0], 0xCAFEBABE),  # source
+      v_mov_b32_e32(v[1], 0x12345678),  # dest (should be preserved)
+      v_permlane64_b32_e32(v[1], v[0]),  # NOP in wave32
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # Dest register should be unchanged (NOP behavior in wave32)
+    self.assertEqual(st.vgpr[0][1], 0x12345678)
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vop2.py

@@ -237,6 +237,32 @@ class TestF16Ops(unittest.TestCase):
     # 2.0 * 3.0 + 1.0 = 7.0, f16 7.0 = 0x4700
     self.assertEqual(result, 0x4700, f"Expected 0x4700 (f16 7.0), got 0x{result:04x}")
 
+  def test_v_max_f16_basic(self):
+    """V_MAX_F16 returns the maximum of two f16 values."""
+    instructions = [
+      s_mov_b32(s[0], 0x3c00),  # f16 1.0
+      s_mov_b32(s[1], 0x4000),  # f16 2.0
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_max_f16_e32(v[2], v[0], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2] & 0xffff
+    self.assertEqual(result, 0x4000, f"Expected 0x4000 (f16 2.0), got 0x{result:04x}")
+
+  def test_v_min_f16_basic(self):
+    """V_MIN_F16 returns the minimum of two f16 values."""
+    instructions = [
+      s_mov_b32(s[0], 0x3c00),  # f16 1.0
+      s_mov_b32(s[1], 0x4000),  # f16 2.0
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_min_f16_e32(v[2], v[0], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2] & 0xffff
+    self.assertEqual(result, 0x3c00, f"Expected 0x3c00 (f16 1.0), got 0x{result:04x}")
+
   def test_v_fmaak_f16_basic(self):
     """V_FMAAK_F16: d = a * b + K."""
     instructions = [
@@ -341,6 +367,293 @@ class TestHiHalfOps(unittest.TestCase):
       self.assertEqual(result, 0x4200, f"Lane {lane}: expected 0x4200, got 0x{result:04x}")
 
 
+class TestVop2F16HiHalf(unittest.TestCase):
+  """Regression tests for VOP2 f16 hi-half operand handling.
+
+  These test the bugs where:
+  1. VOP2 vsrc1 >= 384 (v[128]+) wasn't extracting hi 16 bits
+  2. VOP2 vdst >= 384 (v[128]+) wasn't preserving lo 16 bits
+  """
+
+  def test_v_add_f16_e32_vsrc1_hi_half(self):
+    """V_ADD_F16_E32 with vsrc1 from hi-half (v[128]+).
+
+    When vsrc1 >= 384 (representing v[128]+), the hardware reads from the hi 16 bits
+    of v[vsrc1-128]. The emulator must extract bits [31:16] from the actual VGPR.
+
+    Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v_add_f16_e32 v[1], v[0], v[128]  (vsrc1=v[128] reads hi of v[0])
+      # In VOP2 encoding, vsrc1=384 means v[128], which maps to v[0].hi
+      # v[1] = v[0].lo + v[0].hi = 1.0 + 2.0 = 3.0
+      VOP2(VOP2Op.V_ADD_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    # 1.0 + 2.0 = 3.0, f16 3.0 = 0x4200
+    self.assertEqual(result, 0x4200, f"Expected f16(3.0)=0x4200, got 0x{result:04x}")
+
+  def test_v_mul_f16_e32_vsrc1_hi_half(self):
+    """V_MUL_F16_E32 with vsrc1 from hi-half.
+
+    Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4200_4000: hi=f16(3.0), lo=f16(2.0)
+      s_mov_b32(s[0], 0x42004000),
+      v_mov_b32_e32(v[0], s[0]),
+      # v_mul_f16_e32 v[1], v[0], v[128]  (vsrc1=v[128] reads hi of v[0])
+      # v[1] = v[0].lo * v[0].hi = 2.0 * 3.0 = 6.0
+      VOP2(VOP2Op.V_MUL_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    # 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
+    self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
+
+  def test_v_add_f16_e32_vdst_hi_half(self):
+    """V_ADD_F16_E32 writing to hi-half destination (v[128]+).
+
+    When vdst >= 384 (representing v[128]+), the hardware writes to bits [31:16]
+    of v[vdst-128] while preserving bits [15:0]. The emulator must merge the result.
+
+    Regression test for: VOP2 f16 vdst hi-half write bug.
+    """
+    instructions = [
+      # v[0] = 0x0000_BEEF: lo has marker value
+      s_mov_b32(s[0], 0x0000BEEF),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(1.0), v[2] = f16(2.0)
+      s_mov_b32(s[1], 0x3c00),
+      s_mov_b32(s[2], 0x4000),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]),
+      # v_add_f16_e32 v[128], v[1], v[2]  (vdst=v[128] writes hi of v[0])
+      # v[0].hi = 1.0 + 2.0 = 3.0, v[0].lo preserved = 0xBEEF
+      VOP2(VOP2Op.V_ADD_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    hi = (st.vgpr[0][0] >> 16) & 0xffff
+    lo = st.vgpr[0][0] & 0xffff
+    # hi = 3.0 = 0x4200, lo preserved = 0xBEEF
+    self.assertEqual(hi, 0x4200, f"Expected hi=f16(3.0)=0x4200, got 0x{hi:04x}")
+    self.assertEqual(lo, 0xBEEF, f"Expected lo preserved=0xBEEF, got 0x{lo:04x}")
+
+  def test_v_mul_f16_e32_vdst_hi_half(self):
+    """V_MUL_F16_E32 writing to hi-half destination.
+
+    Regression test for: VOP2 f16 vdst hi-half write bug.
+    """
+    instructions = [
+      # v[0] = 0x0000_DEAD: lo has marker value
+      s_mov_b32(s[0], 0x0000DEAD),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(2.0), v[2] = f16(4.0)
+      s_mov_b32(s[1], 0x4000),
+      s_mov_b32(s[2], 0x4400),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]),
+      # v_mul_f16_e32 v[128], v[1], v[2]  (vdst=v[128] writes hi of v[0])
+      # v[0].hi = 2.0 * 4.0 = 8.0, v[0].lo preserved = 0xDEAD
+      VOP2(VOP2Op.V_MUL_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    hi = (st.vgpr[0][0] >> 16) & 0xffff
+    lo = st.vgpr[0][0] & 0xffff
+    # hi = 8.0 = 0x4800, lo preserved = 0xDEAD
+    self.assertEqual(hi, 0x4800, f"Expected hi=f16(8.0)=0x4800, got 0x{hi:04x}")
+    self.assertEqual(lo, 0xDEAD, f"Expected lo preserved=0xDEAD, got 0x{lo:04x}")
+
+  def test_v_add_f16_e32_both_hi_half(self):
+    """V_ADD_F16_E32 with both vsrc1 and vdst as hi-half (different underlying regs).
+
+    Tests the combination of both fixes: reading vsrc1 from hi-half AND
+    writing result to hi-half destination, using different underlying VGPRs.
+
+    Regression test for: VOP2 f16 hi-half bugs (combined).
+    """
+    instructions = [
+      # v[0] = 0x4000_xxxx: hi=f16(2.0) for vsrc1
+      s_mov_b32(s[0], 0x40000000),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = 0x0000_3c00: lo=f16(1.0) for src0
+      s_mov_b32(s[1], 0x00003c00),
+      v_mov_b32_e32(v[1], s[1]),
+      # v[2] = 0x0000_CAFE: lo=marker for vdst preservation
+      s_mov_b32(s[2], 0x0000CAFE),
+      v_mov_b32_e32(v[2], s[2]),
+      # v_add_f16_e32 v[130], v[1], v[128]
+      # src0 = v[1].lo = 1.0
+      # vsrc1 = v[128] reads v[0].hi = 2.0
+      # result = 1.0 + 2.0 = 3.0
+      # vdst = v[130] writes to v[2].hi, preserving v[2].lo
+      VOP2(VOP2Op.V_ADD_F16, vdst=v[130], src0=v[1], vsrc1=v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    hi = (st.vgpr[0][2] >> 16) & 0xffff
+    lo = st.vgpr[0][2] & 0xffff
+    # hi = 3.0 = 0x4200, lo preserved = 0xCAFE
+    self.assertEqual(hi, 0x4200, f"Expected hi=f16(3.0)=0x4200, got 0x{hi:04x}")
+    self.assertEqual(lo, 0xCAFE, f"Expected lo preserved=0xCAFE, got 0x{lo:04x}")
+
+  def test_v_fmac_f16_e32_vsrc1_hi_half(self):
+    """V_FMAC_F16_E32 with vsrc1 from hi-half.
+
+    V_FMAC_F16: vdst = vdst + src0 * vsrc1
+
+    Regression test for: VOP2 f16 vsrc1 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(3.0) = 0x4200
+      s_mov_b32(s[1], 0x4200),
+      v_mov_b32_e32(v[1], s[1]),
+      # v_fmac_f16_e32 v[1], v[0], v[128]
+      # vdst = v[1] = 3.0 + v[0].lo * v[0].hi = 3.0 + 1.0 * 2.0 = 5.0
+      VOP2(VOP2Op.V_FMAC_F16, vdst=v[1], src0=v[0], vsrc1=v[128]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xffff
+    # 3.0 + 1.0 * 2.0 = 5.0, f16 5.0 = 0x4500
+    self.assertEqual(result, 0x4500, f"Expected f16(5.0)=0x4500, got 0x{result:04x}")
+
+  def test_v_fmac_f16_e32_vdst_hi_half(self):
+    """V_FMAC_F16_E32 writing to hi-half destination.
+
+    V_FMAC_F16: vdst.h = vdst.h + src0 * vsrc1
+
+    When vdst is v[128]+, the accumulator D0 must also read from the hi-half.
+    This tests the bug where D0 was read from lo-half instead of hi-half.
+
+    Regression test for: VOP2 FMAC hi-half D0 accumulator read bug.
+    """
+    instructions = [
+      # v[0] = 0x3800_DEAD: hi=f16(0.5), lo=marker (0xDEAD)
+      s_mov_b32(s[0], 0x3800DEAD),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(2.0) = 0x4000
+      s_mov_b32(s[1], 0x4000),
+      v_mov_b32_e32(v[1], s[1]),
+      # v[2] = f16(3.0) = 0x4200
+      s_mov_b32(s[2], 0x4200),
+      v_mov_b32_e32(v[2], s[2]),
+      # v_fmac_f16_e32 v[128], v[1], v[2]
+      # vdst = v[128] means v[0].hi
+      # D0 = v[0].hi = 0.5
+      # result = D0 + src0 * vsrc1 = 0.5 + 2.0 * 3.0 = 6.5
+      # v[0].hi = 6.5, v[0].lo preserved = 0xDEAD
+      VOP2(VOP2Op.V_FMAC_F16, vdst=v[128], src0=v[1], vsrc1=v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    hi = (st.vgpr[0][0] >> 16) & 0xffff
+    lo = st.vgpr[0][0] & 0xffff
+    # hi = 6.5 = 0x4680, lo preserved = 0xDEAD
+    self.assertEqual(hi, 0x4680, f"Expected hi=f16(6.5)=0x4680, got 0x{hi:04x}")
+    self.assertEqual(lo, 0xDEAD, f"Expected lo preserved=0xDEAD, got 0x{lo:04x}")
+
+  def test_v_mul_f16_e32_src0_hi_half(self):
+    """V_MUL_F16_E32 with src0 from hi-half (src0 >= v[128]).
+
+    When src0 >= 384 (representing v[128]+), the hardware reads from the hi 16 bits
+    of v[src0-128]. The emulator must extract bits [31:16] from the actual VGPR.
+
+    Regression test for: VOP2 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(3.0) = 0x4200
+      s_mov_b32(s[1], 0x4200),
+      v_mov_b32_e32(v[1], s[1]),
+      # v_mul_f16_e32 v[2], v[128], v[1]
+      # src0 = v[128] reads from v[0].hi = 2.0
+      # result = 2.0 * 3.0 = 6.0
+      VOP2(VOP2Op.V_MUL_F16, vdst=v[2], src0=v[128], vsrc1=v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2] & 0xffff
+    # 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
+    self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
+
+  def test_v_add_f16_e32_src0_hi_half(self):
+    """V_ADD_F16_E32 with src0 from hi-half (src0 >= v[128]).
+
+    Regression test for: VOP2 f16 src0 hi-half extraction bug.
+    """
+    instructions = [
+      # v[0] = 0x4000_3c00: hi=f16(2.0), lo=f16(1.0)
+      s_mov_b32(s[0], 0x40003c00),
+      v_mov_b32_e32(v[0], s[0]),
+      # v[1] = f16(5.0) = 0x4500
+      s_mov_b32(s[1], 0x4500),
+      v_mov_b32_e32(v[1], s[1]),
+      # v_add_f16_e32 v[2], v[128], v[1]
+      # src0 = v[128] reads from v[0].hi = 2.0
+      # result = 2.0 + 5.0 = 7.0
+      VOP2(VOP2Op.V_ADD_F16, vdst=v[2], src0=v[128], vsrc1=v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2] & 0xffff
+    # 2.0 + 5.0 = 7.0, f16 7.0 = 0x4700
+    self.assertEqual(result, 0x4700, f"Expected f16(7.0)=0x4700, got 0x{result:04x}")
+
+
+class TestF16InlineConstants(unittest.TestCase):
+  """Regression tests for VOP2 F16 inline float constants.
+
+  For 16-bit VOP2 operations (v_add_f16, v_mul_f16, etc.), inline float constants
+  like 1.0, 2.0 must use F16 encoding (0x3c00, 0x4000) not F32 encoding (0x3f800000).
+
+  The emulator's rsrc() function needs bits=16 to select F16_INLINE constants.
+
+  Regression test for: VOP2 16-bit inline constant using F32 instead of F16.
+  """
+
+  def test_v_add_f16_inline_constant_1_0(self):
+    """V_ADD_F16_E32 with inline constant 1.0 should use F16 encoding."""
+    instructions = [
+      s_mov_b32(s[0], 0x3c00),  # f16 1.0
+      v_mov_b32_e32(v[0], s[0]),
+      # v_add_f16_e32 v[1], 1.0, v[0]  -- 1.0 must be F16 0x3c00, not F32 0x3f800000
+      v_add_f16_e32(v[1], 1.0, v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xFFFF
+    # 1.0 + 1.0 = 2.0, f16 2.0 = 0x4000
+    self.assertEqual(result, 0x4000, f"Expected f16(2.0)=0x4000, got 0x{result:04x}")
+
+  def test_v_add_f16_inline_constant_2_0(self):
+    """V_ADD_F16_E32 with inline constant 2.0."""
+    instructions = [
+      s_mov_b32(s[0], 0x4200),  # f16 3.0
+      v_mov_b32_e32(v[0], s[0]),
+      v_add_f16_e32(v[1], 2.0, v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xFFFF
+    # 2.0 + 3.0 = 5.0, f16 5.0 = 0x4500
+    self.assertEqual(result, 0x4500, f"Expected f16(5.0)=0x4500, got 0x{result:04x}")
+
+  def test_v_mul_f16_inline_constant(self):
+    """V_MUL_F16_E32 with inline constant 2.0."""
+    instructions = [
+      s_mov_b32(s[0], 0x4200),  # f16 3.0
+      v_mov_b32_e32(v[0], s[0]),
+      v_mul_f16_e32(v[1], 2.0, v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][1] & 0xFFFF
+    # 2.0 * 3.0 = 6.0, f16 6.0 = 0x4600
+    self.assertEqual(result, 0x4600, f"Expected f16(6.0)=0x4600, got 0x{result:04x}")
+
+
 class TestCndmask(unittest.TestCase):
   """Tests for V_CNDMASK_B32 and V_CNDMASK_B16."""
 
@@ -447,5 +760,224 @@ class TestSpecialFloatValues(unittest.TestCase):
     self.assertEqual(st.vgpr[0][1], 0x00000000)
 
 
+class TestCarryOps(unittest.TestCase):
+  """Tests for VOP2 carry instructions (v_add_co_ci_u32, v_sub_co_ci_u32, v_subrev_co_ci_u32)."""
+
+  def test_v_subrev_co_ci_u32_no_borrow(self):
+    """V_SUBREV_CO_CI_U32: D0 = S1 - S0 - VCC_IN, when VCC_IN=0."""
+    instructions = [
+      s_mov_b32(VCC_LO, 0),  # VCC = 0 (no borrow in)
+      v_mov_b32_e32(v[0], 5),  # S0 = 5
+      v_mov_b32_e32(v[1], 10),  # S1 = 10
+      v_subrev_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 10 - 5 - 0 = 5
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 5)
+    self.assertEqual(st.vcc, 0)  # No borrow out
+
+  def test_v_subrev_co_ci_u32_with_borrow(self):
+    """V_SUBREV_CO_CI_U32: D0 = S1 - S0 - VCC_IN, when VCC_IN=1."""
+    instructions = [
+      s_mov_b32(VCC_LO, 1),  # VCC = 1 (borrow in)
+      v_mov_b32_e32(v[0], 5),  # S0 = 5
+      v_mov_b32_e32(v[1], 10),  # S1 = 10
+      v_subrev_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 10 - 5 - 1 = 4
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 4)
+    self.assertEqual(st.vcc, 0)  # No borrow out
+
+  def test_v_subrev_co_ci_u32_generates_borrow(self):
+    """V_SUBREV_CO_CI_U32: generates borrow when S0 + VCC_IN > S1."""
+    instructions = [
+      s_mov_b32(VCC_LO, 0),  # VCC = 0
+      v_mov_b32_e32(v[0], 10),  # S0 = 10
+      v_mov_b32_e32(v[1], 5),  # S1 = 5
+      v_subrev_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 5 - 10 - 0 = -5 (underflow)
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 0xFFFFFFFB)  # -5 as unsigned
+    self.assertEqual(st.vcc, 1)  # Borrow out
+
+  def test_v_add_co_ci_u32_no_carry(self):
+    """V_ADD_CO_CI_U32: D0 = S0 + S1 + VCC_IN, when VCC_IN=0."""
+    instructions = [
+      s_mov_b32(VCC_LO, 0),  # VCC = 0 (no carry in)
+      v_mov_b32_e32(v[0], 5),  # S0 = 5
+      v_mov_b32_e32(v[1], 10),  # S1 = 10
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 5 + 10 + 0 = 15
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 15)
+    self.assertEqual(st.vcc, 0)  # No carry out
+
+  def test_v_add_co_ci_u32_with_carry(self):
+    """V_ADD_CO_CI_U32: D0 = S0 + S1 + VCC_IN, when VCC_IN=1."""
+    instructions = [
+      s_mov_b32(VCC_LO, 1),  # VCC = 1 (carry in)
+      v_mov_b32_e32(v[0], 5),  # S0 = 5
+      v_mov_b32_e32(v[1], 10),  # S1 = 10
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 5 + 10 + 1 = 16
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 16)
+    self.assertEqual(st.vcc, 0)  # No carry out
+
+  def test_v_add_co_ci_u32_generates_carry(self):
+    """V_ADD_CO_CI_U32: generates carry when overflow occurs."""
+    instructions = [
+      s_mov_b32(VCC_LO, 1),  # VCC = 1 (carry in)
+      s_mov_b32(s[0], 0xFFFFFFFF),  # max u32
+      v_mov_b32_e32(v[0], s[0]),  # S0 = 0xFFFFFFFF
+      v_mov_b32_e32(v[1], 0),  # S1 = 0
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 0xFFFFFFFF + 0 + 1 = 0 (overflow)
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 0)  # Overflowed to 0
+    self.assertEqual(st.vcc, 1)  # Carry out
+
+  def test_v_add_co_ci_u32_clears_carry(self):
+    """V_ADD_CO_CI_U32: VCC must be updated even when no carry is generated.
+
+    This tests the case where VCC=1 going in (carry-in consumed) but the addition
+    does not overflow, so VCC must be cleared to 0.
+
+    Regression test for: VCC not being written by v_add_co_ci_u32_e32.
+    """
+    instructions = [
+      s_mov_b32(VCC_LO, 1),  # VCC = 1 (carry in)
+      v_mov_b32_e32(v[0], 1),  # S0 = 1
+      v_mov_b32_e32(v[1], 1),  # S1 = 1
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 1 + 1 + 1 = 3 (no overflow)
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 3)  # 1 + 1 + 1 = 3
+    self.assertEqual(st.vcc, 0)  # No carry out - VCC must be cleared
+
+  def test_v_add_co_ci_u32_multilane_clears_vcc(self):
+    """V_ADD_CO_CI_U32 with multiple lanes: VCC bits must be updated per-lane.
+
+    When VCC has multiple bits set (one per active lane), and the addition doesn't
+    overflow for any lane, all VCC bits must be cleared.
+
+    Regression test for: VCC not being written by v_add_co_ci_u32_e32 in multi-lane case.
+    """
+    instructions = [
+      s_mov_b32(VCC_LO, 0b11),  # VCC = 0b11 (lanes 0,1 have carry-in)
+      v_mov_b32_e32(v[0], 1),  # S0 = 1 for all lanes
+      v_mov_b32_e32(v[1], 1),  # S1 = 1 for all lanes
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 1 + 1 + 1 = 3 (no overflow)
+    ]
+    st = run_program(instructions, n_lanes=2)
+    self.assertEqual(st.vgpr[0][2], 3)  # lane 0: 1 + 1 + 1 = 3
+    self.assertEqual(st.vgpr[1][2], 3)  # lane 1: 1 + 1 + 1 = 3
+    self.assertEqual(st.vcc, 0)  # No carry out for any lane - all VCC bits must be cleared
+
+  def test_v_add_co_ci_u32_preserves_inactive_vcc_bits(self):
+    """V_ADD_CO_CI_U32: VCC carry-out overwrites entire VCC register.
+
+    VOP2 carry instructions write ALL VCC bits based on carry-out, clearing
+    bits for lanes that don't overflow regardless of EXEC mask.
+
+    Note: This differs from VOPC which only writes active lane bits.
+    """
+    instructions = [
+      s_mov_b32(VCC_LO, 0x00010000),  # VCC bit 16 set
+      v_mov_b32_e32(v[0], 1),  # S0 = 1
+      v_mov_b32_e32(v[1], 1),  # S1 = 1
+      v_add_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 1 + 1 + 0 = 2 (no carry)
+    ]
+    st = run_program(instructions, n_lanes=4)
+    self.assertEqual(st.vgpr[0][2], 2)  # lane 0: 1 + 1 + 0 = 2
+    # VCC should be completely cleared (all lanes have no carry-out)
+    self.assertEqual(st.vcc, 0)
+
+  def test_v_add_co_ci_u32_all_lanes_same_result(self):
+    """V_ADD_CO_CI_U32: all active lanes should produce the same result.
+
+    When the same constant inputs are used across all lanes, each lane should
+    compute the same result and write to its own VGPR slot.
+
+    Regression test for: VGPR writes not happening for all lanes.
+    """
+    instructions = [
+      s_mov_b32(VCC_LO, 0),  # No carry-in
+      v_mov_b32_e32(v[0], 3),  # inline constant 3
+      v_mov_b32_e32(v[1], 5),  # value 5
+      v_add_co_ci_u32_e32(v[1], 3, v[1]),  # v[1] = 3 + v[1] + 0 = 3 + 5 = 8
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # All 4 lanes should have v[1] = 8
+    for lane in range(4):
+      self.assertEqual(st.vgpr[lane][1], 8, f"lane {lane} should have v[1]=8")
+
+  def test_v_sub_co_ci_u32_no_borrow(self):
+    """V_SUB_CO_CI_U32: D0 = S0 - S1 - VCC_IN, when VCC_IN=0."""
+    instructions = [
+      s_mov_b32(VCC_LO, 0),  # VCC = 0 (no borrow in)
+      v_mov_b32_e32(v[0], 10),  # S0 = 10
+      v_mov_b32_e32(v[1], 5),  # S1 = 5
+      v_sub_co_ci_u32_e32(v[2], v[0], v[1]),  # D0 = 10 - 5 - 0 = 5
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 5)
+    self.assertEqual(st.vcc, 0)  # No borrow out
+
+  def test_v_sub_co_ci_u32_vop3sd_separate_carry_regs(self):
+    """VOP3SD V_SUB_CO_CI_U32: carry-in from src2, carry-out to sdst (separate registers).
+
+    This tests the VOP3SD encoding where src2 specifies the carry-in register
+    independently from sdst (carry-out). The bug was reading carry-in from sdst
+    instead of src2.
+
+    Computation: D0 = S0 - S1 - carry_in = 0 - 0 - 1 = -1 = 0xFFFFFFFF
+    """
+    instructions = [
+      s_mov_b32(s[6], 1),  # carry-in = 1 (in s[6])
+      s_mov_b32(s[10], 0),  # carry-out dest = 0 initially (in s[10])
+      # VOP3SD: v_sub_co_ci_u32(vdst, sdst, src0, src1, src2)
+      # src2 is carry-in (s[6]=1), sdst is carry-out (s[10])
+      v_sub_co_ci_u32(v[0], s[10], 0, 0, s[6]),  # D0 = 0 - 0 - 1 = -1
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0xFFFFFFFF)  # -1 as unsigned
+    self.assertEqual(st.sgpr[10], 1)  # Borrow out to s[10]
+
+  def test_v_add_co_ci_u32_vop3sd_separate_carry_regs(self):
+    """VOP3SD V_ADD_CO_CI_U32: carry-in from src2, carry-out to sdst (separate registers).
+
+    This tests the VOP3SD encoding where src2 specifies the carry-in register
+    independently from sdst (carry-out).
+
+    Computation: D0 = S0 + S1 + carry_in = 5 + 10 + 1 = 16
+    """
+    instructions = [
+      s_mov_b32(s[6], 1),  # carry-in = 1 (in s[6])
+      s_mov_b32(s[10], 0),  # carry-out dest = 0 initially (in s[10])
+      # VOP3SD: v_add_co_ci_u32(vdst, sdst, src0, src1, src2)
+      v_add_co_ci_u32(v[0], s[10], 5, 10, s[6]),  # D0 = 5 + 10 + 1 = 16
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 16)
+    self.assertEqual(st.sgpr[10], 0)  # No carry out
+
+  def test_v_add_co_ci_u32_vop3sd_null_sdst(self):
+    """VOP3SD V_ADD_CO_CI_U32 with sdst=NULL: carry output is discarded.
+
+    When sdst=NULL (register 124), the carry-out should NOT be written anywhere.
+    We verify this by checking that VCC (which we set to a sentinel value) is unchanged.
+    """
+    instructions = [
+      s_mov_b32(VCC_LO, 0xDEADBEEF),  # Sentinel value in VCC
+      s_mov_b32(s[6], 0),  # carry-in = 0
+      # VOP3SD with NULL sdst: carry-out should be discarded
+      # Uses 0xFFFFFFFF + 1 + 0 = 0 with carry-out=1, but carry should not be written
+      v_add_co_ci_u32(v[0], NULL, 0xFFFFFFFF, 1, s[6]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][0], 0)  # 0xFFFFFFFF + 1 + 0 = 0 (overflow)
+    self.assertEqual(st.vcc, 0xDEADBEEF)  # VCC unchanged - carry was discarded
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vop3p.py

@@ -149,7 +149,6 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_src2_f16_lo(self):
     """V_FMA_MIX_F32 with src2 as f16 from lo bits."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_2 = f32_to_f16(2.0)
     instructions = [
       s_mov_b32(s[0], f2i(1.0)),
@@ -166,7 +165,6 @@ class TestFmaMix(unittest.TestCase):
 
   def test_v_fma_mix_f32_src2_f16_hi(self):
     """V_FMA_MIX_F32 with src2 as f16 from hi bits."""
-    from extra.assembly.amd.pcode import f32_to_f16
     f16_2 = f32_to_f16(2.0)
     val = (f16_2 << 16) | 0
     instructions = [
@@ -197,9 +195,60 @@ class TestFmaMix(unittest.TestCase):
     result = i2f(st.vgpr[0][3])
     self.assertAlmostEqual(result, 7.0, places=5)
 
+  def test_v_fma_mix_f32_with_abs_f16_src2_lo(self):
+    """V_FMA_MIX_F32 with abs modifier on f16 src2 (lo half). Regression test for sin(1.0) bug."""
+    f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
+    instructions = [
+      s_mov_b32(s[0], f2i(0.0)),  # src0 = 0.0 (f32)
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[1], f2i(1.0)),  # src1 = 1.0 (f32)
+      v_mov_b32_e32(v[1], s[1]),
+      s_mov_b32(s[2], f16_neg1),  # src2 = -1.0 (f16 in lo)
+      v_mov_b32_e32(v[2], s[2]),
+      # 0*1 + abs(-1.0) = 1.0; neg_hi=4 means abs on src2, opsel_hi2=1 means src2 is f16
+      VOP3P(VOP3POp.V_FMA_MIX_F32, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=1, neg_hi=4),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, 1.0, places=5)
+
+  def test_v_fma_mix_f32_with_neg_f16_src2_lo(self):
+    """V_FMA_MIX_F32 with neg modifier on f16 src2 (lo half)."""
+    f16_1 = f32_to_f16(1.0)  # 0x3c00
+    instructions = [
+      s_mov_b32(s[0], f2i(0.0)),  # src0 = 0.0 (f32)
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[1], f2i(1.0)),  # src1 = 1.0 (f32)
+      v_mov_b32_e32(v[1], s[1]),
+      s_mov_b32(s[2], f16_1),  # src2 = 1.0 (f16 in lo)
+      v_mov_b32_e32(v[2], s[2]),
+      # 0*1 + neg(1.0) = -1.0; neg=4 means neg on src2, opsel_hi2=1 means src2 is f16
+      VOP3P(VOP3POp.V_FMA_MIX_F32, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=1, neg=4),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, -1.0, places=5)
+
+  def test_v_fma_mix_f32_with_abs_f16_src2_hi(self):
+    """V_FMA_MIX_F32 with abs modifier on f16 src2 (hi half)."""
+    f16_neg1 = f32_to_f16(-1.0)  # 0xbc00
+    val = (f16_neg1 << 16) | 0  # -1.0 in hi, 0 in lo
+    instructions = [
+      s_mov_b32(s[0], f2i(0.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      s_mov_b32(s[1], f2i(1.0)),
+      v_mov_b32_e32(v[1], s[1]),
+      s_mov_b32(s[2], val),
+      v_mov_b32_e32(v[2], s[2]),
+      # opsel=4 selects hi half of src2; neg_hi=4 means abs on src2
+      VOP3P(VOP3POp.V_FMA_MIX_F32, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=4, opsel_hi=0, opsel_hi2=1, neg_hi=4),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, 1.0, places=5)
+
   def test_v_fma_mixlo_f16(self):
     """V_FMA_MIXLO_F16 writes to low 16 bits of destination."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], f2i(2.0)),
       v_mov_b32_e32(v[0], s[0]),
@@ -212,14 +261,13 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = _f16(st.vgpr[0][3] & 0xffff)
+    lo = f16(st.vgpr[0][3] & 0xffff)
     hi = (st.vgpr[0][3] >> 16) & 0xffff
     self.assertAlmostEqual(lo, 7.0, places=1)
     self.assertEqual(hi, 0xdead, f"hi should be preserved, got 0x{hi:04x}")
 
   def test_v_fma_mixlo_f16_all_f32_sources(self):
     """V_FMA_MIXLO_F16 with all f32 sources."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], f2i(1.0)),
       v_mov_b32_e32(v[0], s[0]),
@@ -231,13 +279,12 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[0], src1=v[1], src2=v[2], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = _f16(st.vgpr[0][3] & 0xffff)
+    lo = f16(st.vgpr[0][3] & 0xffff)
     # 1*2+3 = 5
     self.assertAlmostEqual(lo, 5.0, places=1)
 
   def test_v_fma_mixlo_f16_sin_case(self):
     """V_FMA_MIXLO_F16 case from sin kernel."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x3f800000),  # f32 1.0
       v_mov_b32_e32(v[3], s[0]),
@@ -250,7 +297,7 @@ class TestFmaMix(unittest.TestCase):
       VOP3P(VOP3POp.V_FMA_MIXLO_F16, vdst=v[3], src0=v[3], src1=s[6], src2=v[5], opsel=0, opsel_hi=0, opsel_hi2=0),
     ]
     st = run_program(instructions, n_lanes=1)
-    lo = _f16(st.vgpr[0][3] & 0xffff)
+    lo = f16(st.vgpr[0][3] & 0xffff)
     self.assertAlmostEqual(lo, -3.14159, delta=0.01)
 
 
@@ -259,7 +306,6 @@ class TestVOP3P(unittest.TestCase):
 
   def test_v_pk_add_f16_basic(self):
     """V_PK_ADD_F16 adds two packed f16 values."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x40003c00),  # hi=2.0, lo=1.0
       s_mov_b32(s[1], 0x44004200),  # hi=4.0, lo=3.0
@@ -269,14 +315,13 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 4.0, places=2)
     self.assertAlmostEqual(hi, 6.0, places=2)
 
   def test_v_pk_mul_f16_basic(self):
     """V_PK_MUL_F16 multiplies two packed f16 values."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x42004000),  # hi=3.0, lo=2.0
       s_mov_b32(s[1], 0x45004400),  # hi=5.0, lo=4.0
@@ -286,14 +331,13 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 8.0, places=1)
     self.assertAlmostEqual(hi, 15.0, places=1)
 
   def test_v_pk_fma_f16_basic(self):
     """V_PK_FMA_F16: D = A * B + C for packed f16."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x42004000),  # A: hi=3.0, lo=2.0
       s_mov_b32(s[1], 0x45004400),  # B: hi=5.0, lo=4.0
@@ -305,8 +349,8 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][3]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 9.0, places=1)   # 2*4+1
     self.assertAlmostEqual(hi, 16.0, places=0)  # 3*5+1
 
@@ -315,7 +359,6 @@ class TestVOP3P(unittest.TestCase):
     Inline constants for VOP3P are f16 values in the low 16 bits only.
     hi half of inline constant is 0, so hi result = v0.hi + 0 = 1.0.
     """
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x3c003c00),  # packed f16: hi=1.0, lo=1.0
       v_mov_b32_e32(v[0], s[0]),
@@ -323,8 +366,8 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     # lo = 1.0 + 1.0 = 2.0, hi = 1.0 + 0.0 = 1.0 (inline const hi half is 0)
     self.assertAlmostEqual(lo, 2.0, places=2)
     self.assertAlmostEqual(hi, 1.0, places=2)
@@ -333,7 +376,6 @@ class TestVOP3P(unittest.TestCase):
     """V_PK_MUL_F16 with inline constant POS_TWO (2.0).
     Inline constant has value only in low 16 bits, hi is 0.
     """
-    from extra.assembly.amd.pcode import _f16
     # v0 = packed (3.0, 4.0), multiply by POS_TWO
     # lo = 3.0 * 2.0 = 6.0, hi = 4.0 * 0.0 = 0.0 (inline const hi is 0)
     instructions = [
@@ -343,14 +385,102 @@ class TestVOP3P(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][1]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 6.0, places=1)
     self.assertAlmostEqual(hi, 0.0, places=1)
 
 
+class TestWMMAF16(unittest.TestCase):
+  """Tests for WMMA F16 output variant (V_WMMA_F16_16X16X16_F16).
+
+  Note: RDNA3 WMMA F16 uses 8 VGPRs for accumulator/output (same as F32 variant),
+  but values are packed as f16. This differs from RDNA4 which uses 4 VGPRs.
+  """
+
+  def test_v_wmma_f16_16x16x16_f16_all_ones(self):
+    """V_WMMA_F16_16X16X16_F16 with all ones produces 16.0 in f16."""
+    instructions = []
+    instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
+    # Initialize A matrix in v[16:23] (8 regs)
+    for i in range(16, 24):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize B matrix in v[24:31] (8 regs)
+    for i in range(24, 32):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize C (accumulator) in v[0:7] to zero (8 regs for RDNA3 WMMA F16)
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], 0))
+    # WMMA F16: D = A @ B + C
+    instructions.append(v_wmma_f16_16x16x16_f16(v[0:7], v[16:23], v[24:31], v[0:7]))
+    st = run_program(instructions, n_lanes=32)
+    # Result should be 16.0 in f16, stored in lo 16 bits of each VGPR (hi bits are 0)
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        lo = f16(result & 0xffff)
+        self.assertAlmostEqual(lo, 16.0, places=1, msg=f"v[{reg}] lane {lane}: expected 16.0, got {lo}")
+        self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
+
+  def test_v_wmma_f16_16x16x16_f16_with_accumulator(self):
+    """V_WMMA_F16_16X16X16_F16 with non-zero accumulator."""
+    instructions = []
+    instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
+    instructions.append(s_mov_b32(s[1], 0x4500))  # f16 5.0 in lo bits only
+    # Initialize A matrix in v[16:23] (8 regs)
+    for i in range(16, 24):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize B matrix in v[24:31] (8 regs)
+    for i in range(24, 32):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize C (accumulator) in v[0:7] to 5.0 in lo bits (8 regs for RDNA3 WMMA F16)
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], s[1]))
+    # WMMA F16: D = A @ B + C
+    instructions.append(v_wmma_f16_16x16x16_f16(v[0:7], v[16:23], v[24:31], v[0:7]))
+    st = run_program(instructions, n_lanes=32)
+    # Result should be 16.0 + 5.0 = 21.0 in f16, stored in lo 16 bits (hi bits are 0)
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        lo = f16(result & 0xffff)
+        self.assertAlmostEqual(lo, 21.0, places=0, msg=f"v[{reg}] lane {lane}: expected 21.0, got {lo}")
+        self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
+
+  def test_v_wmma_f16_16x16x16_f16_high_registers(self):
+    """V_WMMA_F16_16X16X16_F16 with high register indices.
+
+    Regression test: WMMA was using static register indices instead of dynamic.
+    This test uses v[64:71] for A, v[80:87] for B, v[96:103] for C/D.
+    """
+    instructions = []
+    instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
+    # Initialize A matrix in v[64:71] (8 regs)
+    for i in range(64, 72):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize B matrix in v[80:87] (8 regs)
+    for i in range(80, 88):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize C (accumulator) in v[96:103] to zero (8 regs for RDNA3 WMMA F16)
+    for i in range(96, 104):
+      instructions.append(v_mov_b32_e32(v[i], 0))
+    # WMMA F16: D = A @ B + C, result in v[96:103]
+    instructions.append(v_wmma_f16_16x16x16_f16(v[96:103], v[64:71], v[80:87], v[96:103]))
+    # Copy results to v[0:7] for checking
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], v[96+i]))
+    st = run_program(instructions, n_lanes=32)
+    # Result should be 16.0 in f16, stored in lo 16 bits (hi bits are 0)
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        lo = f16(result & 0xffff)
+        self.assertAlmostEqual(lo, 16.0, places=1, msg=f"v[{reg}] lane {lane}: expected 16.0, got {lo}")
+        self.assertEqual(result >> 16, 0, msg=f"v[{reg}] lane {lane}: hi bits should be 0")
+
+
 class TestWMMA(unittest.TestCase):
-  """Tests for WMMA (Wave Matrix Multiply-Accumulate) instructions."""
+  """Tests for WMMA (Wave Matrix Multiply-Accumulate) instructions with F32 output."""
 
   def test_v_wmma_f32_16x16x16_f16_all_ones(self):
     """V_WMMA_F32_16X16X16_F16 with all ones produces 16.0."""
@@ -385,6 +515,75 @@ class TestWMMA(unittest.TestCase):
         result = st.vgpr[lane][reg]
         self.assertEqual(result, expected, f"v[{reg}] lane {lane}: expected 21.0, got {i2f(result)}")
 
+  def test_v_wmma_f32_16x16x16_f16_high_registers(self):
+    """V_WMMA_F32_16X16X16_F16 with high register indices.
+
+    Regression test: WMMA was using static register indices instead of dynamic,
+    causing incorrect results when registers weren't at the default positions.
+    This test uses v[64:71] for A, v[80:87] for B, v[96:103] for C/D.
+    """
+    instructions = []
+    instructions.append(s_mov_b32(s[0], 0x3c003c00))  # packed f16 1.0
+    # Initialize A matrix in v[64:71]
+    for i in range(64, 72):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize B matrix in v[80:87]
+    for i in range(80, 88):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    # Initialize C (accumulator) in v[96:103] to zero
+    for i in range(96, 104):
+      instructions.append(v_mov_b32_e32(v[i], 0))
+    # WMMA: D = A @ B + C, result in v[96:103]
+    instructions.append(v_wmma_f32_16x16x16_f16(v[96:103], v[64:71], v[80:87], v[96:103]))
+    # Copy results to v[0:7] for checking
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], v[96+i]))
+    st = run_program(instructions, n_lanes=32)
+    expected = f2i(16.0)
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        self.assertEqual(result, expected, f"v[{reg}] lane {lane}: expected 16.0, got {i2f(result)}")
+
+
+class TestWMMABF16(unittest.TestCase):
+  """Tests for WMMA BF16 instructions."""
+
+  def test_v_wmma_f32_16x16x16_bf16_all_ones(self):
+    """V_WMMA_F32_16X16X16_BF16 with all ones produces 16.0."""
+    instructions = []
+    # BF16 1.0 = 0x3f80, packed = 0x3f803f80
+    instructions.append(s_mov_b32(s[0], 0x3f803f80))
+    for i in range(16, 32):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], 0))
+    instructions.append(v_wmma_f32_16x16x16_bf16(v[0:7], v[16:23], v[24:31], v[0:7]))
+    st = run_program(instructions, n_lanes=32)
+    expected = f2i(16.0)
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        self.assertEqual(result, expected, f"v[{reg}] lane {lane}: expected 16.0, got {i2f(result)}")
+
+  def test_v_wmma_f32_16x16x16_bf16_with_accumulator(self):
+    """V_WMMA_F32_16X16X16_BF16 with non-zero accumulator."""
+    instructions = []
+    # BF16 1.0 = 0x3f80, packed = 0x3f803f80
+    instructions.append(s_mov_b32(s[0], 0x3f803f80))
+    instructions.append(s_mov_b32(s[1], f2i(5.0)))
+    for i in range(16, 32):
+      instructions.append(v_mov_b32_e32(v[i], s[0]))
+    for i in range(8):
+      instructions.append(v_mov_b32_e32(v[i], s[1]))
+    instructions.append(v_wmma_f32_16x16x16_bf16(v[0:7], v[16:23], v[24:31], v[0:7]))
+    st = run_program(instructions, n_lanes=32)
+    expected = f2i(21.0)  # 16 + 5
+    for lane in range(32):
+      for reg in range(8):
+        result = st.vgpr[lane][reg]
+        self.assertEqual(result, expected, f"v[{reg}] lane {lane}: expected 21.0, got {i2f(result)}")
+
 
 class TestSpecialOps(unittest.TestCase):
   """Tests for special operations (SAD, PERM, DOT2)."""
@@ -498,7 +697,6 @@ class TestPackedMixedSigns(unittest.TestCase):
 
   def test_pk_add_f16_mixed_signs(self):
     """V_PK_ADD_F16 with mixed positive/negative values."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0xc0003c00),  # packed: hi=-2.0, lo=1.0
       s_mov_b32(s[1], 0x3c003c00),  # packed: hi=1.0, lo=1.0
@@ -508,14 +706,13 @@ class TestPackedMixedSigns(unittest.TestCase):
     ]
     st = run_program(instructions, n_lanes=1)
     result = st.vgpr[0][2]
-    lo = _f16(result & 0xffff)
-    hi = _f16((result >> 16) & 0xffff)
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
     self.assertAlmostEqual(lo, 2.0, places=2)   # 1.0 + 1.0
     self.assertAlmostEqual(hi, -1.0, places=2)  # -2.0 + 1.0
 
   def test_pk_mul_f16_zero(self):
     """V_PK_MUL_F16 with zero."""
-    from extra.assembly.amd.pcode import _f16
     instructions = [
       s_mov_b32(s[0], 0x40004000),  # packed: 2.0, 2.0
       s_mov_b32(s[1], 0x00000000),  # packed: 0.0, 0.0
@@ -528,5 +725,277 @@ class TestPackedMixedSigns(unittest.TestCase):
     self.assertEqual(result, 0x00000000, "2.0 * 0.0 should be 0.0")
 
 
+class TestDot2F32F16(unittest.TestCase):
+  """Tests for V_DOT2_F32_F16 - dot product of f16 pairs producing f32."""
+
+  def test_v_dot2_f32_f16_basic(self):
+    """V_DOT2_F32_F16: dot product of two packed f16 pairs -> f32."""
+    # src0 = {hi=2.0, lo=1.0}, src1 = {hi=4.0, lo=3.0}
+    # result = 1.0*3.0 + 2.0*4.0 + 0 = 3 + 8 = 11.0
+    src0 = (f32_to_f16(2.0) << 16) | f32_to_f16(1.0)
+    src1 = (f32_to_f16(4.0) << 16) | f32_to_f16(3.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, 11.0, places=2)
+
+  def test_v_dot2_f32_f16_with_accumulator(self):
+    """V_DOT2_F32_F16 with non-zero f32 accumulator."""
+    # src0 = {hi=1.0, lo=1.0}, src1 = {hi=1.0, lo=1.0}, acc = 5.0
+    # result = 1.0*1.0 + 1.0*1.0 + 5.0 = 7.0
+    src0 = (f32_to_f16(1.0) << 16) | f32_to_f16(1.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], f2i(5.0)),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[0]),  # same as src0
+      v_mov_b32_e32(v[2], s[1]),
+      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, 7.0, places=2)
+
+  def test_v_dot2_f32_f16_negative_values(self):
+    """V_DOT2_F32_F16 with negative f16 values."""
+    # src0 = {hi=-2.0, lo=3.0}, src1 = {hi=1.0, lo=2.0}
+    # result = 3.0*2.0 + (-2.0)*1.0 + 0 = 6 - 2 = 4.0
+    # NOTE: Hardware DOT2 may have up to 1 ULP difference due to internal implementation
+    src0 = (f32_to_f16(-2.0) << 16) | f32_to_f16(3.0)
+    src1 = (f32_to_f16(1.0) << 16) | f32_to_f16(2.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot2_f32_f16(v[3], v[0], v[1], v[2], opsel_hi=3, opsel_hi2=1),
+    ]
+    st = run_program(instructions, n_lanes=1, ulp_tolerance=1)
+    result = i2f(st.vgpr[0][3])
+    self.assertAlmostEqual(result, 4.0, places=2)
+
+
+class TestDot2F16F16(unittest.TestCase):
+  """Tests for V_DOT2_F16_F16 - dot product of f16 pairs producing f16."""
+
+  def test_v_dot2_f16_f16_basic(self):
+    """V_DOT2_F16_F16: dot product of two packed f16 pairs -> f16."""
+    # src0 = {hi=2.0, lo=1.0}, src1 = {hi=3.0, lo=2.0}
+    # result = 1.0*2.0 + 2.0*3.0 + 0 = 2 + 6 = 8.0 (f16)
+    src0 = (f32_to_f16(2.0) << 16) | f32_to_f16(1.0)
+    src1 = (f32_to_f16(3.0) << 16) | f32_to_f16(2.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot2_f16_f16(v[3], v[0], v[1], v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = f16(st.vgpr[0][3] & 0xffff)
+    self.assertAlmostEqual(result, 8.0, places=1)
+
+  def test_v_dot2_f16_f16_with_accumulator(self):
+    """V_DOT2_F16_F16 with non-zero f16 accumulator."""
+    # src0 = {hi=1.0, lo=1.0}, src1 = {hi=1.0, lo=1.0}, acc = 3.0 (f16)
+    # result = 1.0*1.0 + 1.0*1.0 + 3.0 = 5.0 (f16)
+    src0 = (f32_to_f16(1.0) << 16) | f32_to_f16(1.0)
+    acc = f32_to_f16(3.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[2], acc),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[0]),  # same as src0
+      v_mov_b32_e32(v[2], s[2]),
+      v_dot2_f16_f16(v[3], v[0], v[1], v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = f16(st.vgpr[0][3] & 0xffff)
+    self.assertAlmostEqual(result, 5.0, places=1)
+
+
+class TestSignedDotProducts(unittest.TestCase):
+  """Tests for V_DOT4_I32_IU8 and V_DOT8_I32_IU4 with signed inputs."""
+
+  def test_v_dot4_i32_iu8_signed_both(self):
+    """V_DOT4_I32_IU8 with both inputs signed (neg=0b011)."""
+    # src0 = {-1, -2, 3, 4} as i8 = {0xff, 0xfe, 0x03, 0x04}
+    # src1 = {1, 1, 1, 1} as i8
+    # result = (-1)*1 + (-2)*1 + 3*1 + 4*1 = -1 - 2 + 3 + 4 = 4
+    src0 = (0xff << 24) | (0xfe << 16) | (0x03 << 8) | 0x04  # -1, -2, 3, 4
+    src1 = 0x01010101  # 1, 1, 1, 1
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][3]
+    # Result is i32, interpret as signed
+    if result >= 0x80000000:
+      result = result - 0x100000000
+    self.assertEqual(result, 4)
+
+  def test_v_dot4_i32_iu8_src0_signed(self):
+    """V_DOT4_I32_IU8 with only src0 signed (neg=0b001)."""
+    # src0 = {-1, -1, -1, -1} as i8 = {0xff, 0xff, 0xff, 0xff}
+    # src1 = {2, 2, 2, 2} as u8
+    # result = (-1)*2 + (-1)*2 + (-1)*2 + (-1)*2 = -8
+    src0 = 0xffffffff  # -1, -1, -1, -1 (as i8)
+    src1 = 0x02020202  # 2, 2, 2, 2 (as u8)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b001),  # src0 signed
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][3]
+    if result >= 0x80000000:
+      result = result - 0x100000000
+    self.assertEqual(result, -8)
+
+  def test_v_dot4_i32_iu8_src1_signed(self):
+    """V_DOT4_I32_IU8 with only src1 signed (neg=0b010)."""
+    # src0 = {2, 2, 2, 2} as u8
+    # src1 = {-1, -1, -1, -1} as i8 = {0xff, 0xff, 0xff, 0xff}
+    # result = 2*(-1) + 2*(-1) + 2*(-1) + 2*(-1) = -8
+    src0 = 0x02020202  # 2, 2, 2, 2 (as u8)
+    src1 = 0xffffffff  # -1, -1, -1, -1 (as i8)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0b010),  # src1 signed
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][3]
+    if result >= 0x80000000:
+      result = result - 0x100000000
+    self.assertEqual(result, -8)
+
+  def test_v_dot4_i32_iu8_unsigned_as_reference(self):
+    """V_DOT4_I32_IU8 with both unsigned (neg=0) - same as V_DOT4_U32_U8."""
+    # src0 = {0xff, 0xff, 0xff, 0xff} = 255 each as u8
+    # src1 = {1, 1, 1, 1}
+    # result = 255*1 + 255*1 + 255*1 + 255*1 = 1020
+    src0 = 0xffffffff
+    src1 = 0x01010101
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot4_i32_iu8(v[3], v[0], v[1], v[2], neg=0),  # both unsigned
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][3], 1020)
+
+  def test_v_dot8_i32_iu4_signed_both(self):
+    """V_DOT8_I32_IU4 with both inputs signed (neg=0b011)."""
+    # src0 = 8 nibbles: {-1, -2, 3, 4, -1, -2, 3, 4} as i4
+    # i4 -1 = 0xf, -2 = 0xe, 3 = 0x3, 4 = 0x4
+    # src0 = 0xfe34fe34
+    # src1 = {1, 1, 1, 1, 1, 1, 1, 1} as i4 = 0x11111111
+    # result = 2 * ((-1)*1 + (-2)*1 + 3*1 + 4*1) = 2 * 4 = 8
+    src0 = 0xfe34fe34
+    src1 = 0x11111111
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot8_i32_iu4(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][3]
+    if result >= 0x80000000:
+      result = result - 0x100000000
+    self.assertEqual(result, 8)
+
+  def test_v_dot8_i32_iu4_all_negative(self):
+    """V_DOT8_I32_IU4 with all negative signed values."""
+    # src0 = 8 nibbles all -1 (0xf) = 0xffffffff
+    # src1 = 8 nibbles all 1 = 0x11111111
+    # result = 8 * ((-1)*1) = -8
+    src0 = 0xffffffff  # all -1 as i4
+    src1 = 0x11111111  # all 1
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], 0),
+      v_dot8_i32_iu4(v[3], v[0], v[1], v[2], neg=0b011),  # both signed
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][3]
+    if result >= 0x80000000:
+      result = result - 0x100000000
+    self.assertEqual(result, -8)
+
+
+class TestPkMinMaxF16(unittest.TestCase):
+  """Tests for V_PK_MIN_F16 and V_PK_MAX_F16."""
+
+  def test_v_pk_min_f16_basic(self):
+    """V_PK_MIN_F16: packed min of two f16 pairs."""
+    # src0 = {hi=3.0, lo=1.0}, src1 = {hi=2.0, lo=4.0}
+    # result = {min(3,2)=2, min(1,4)=1}
+    src0 = (f32_to_f16(3.0) << 16) | f32_to_f16(1.0)
+    src1 = (f32_to_f16(2.0) << 16) | f32_to_f16(4.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_pk_min_f16(v[2], v[0], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2]
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
+    self.assertAlmostEqual(lo, 1.0, delta=0.01)
+    self.assertAlmostEqual(hi, 2.0, delta=0.01)
+
+  def test_v_pk_max_f16_basic(self):
+    """V_PK_MAX_F16: packed max of two f16 pairs."""
+    # src0 = {hi=3.0, lo=1.0}, src1 = {hi=2.0, lo=4.0}
+    # result = {max(3,2)=3, max(1,4)=4}
+    src0 = (f32_to_f16(3.0) << 16) | f32_to_f16(1.0)
+    src1 = (f32_to_f16(2.0) << 16) | f32_to_f16(4.0)
+    instructions = [
+      s_mov_b32(s[0], src0),
+      s_mov_b32(s[1], src1),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[1]),
+      v_pk_max_f16(v[2], v[0], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    result = st.vgpr[0][2]
+    lo = f16(result & 0xffff)
+    hi = f16((result >> 16) & 0xffff)
+    self.assertAlmostEqual(lo, 4.0, delta=0.01)
+    self.assertAlmostEqual(hi, 3.0, delta=0.01)
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vopc.py

@@ -324,6 +324,29 @@ class TestCmpInt(unittest.TestCase):
     st = run_program(instructions, n_lanes=4)
     self.assertEqual(st.vcc & 0xf, 0xf, "All lanes should match")
 
+  def test_v_cmp_ne_u32_with_zero(self):
+    """V_CMP_NE_U32: compare with zero, used for int->bool cast."""
+    instructions = [
+      v_mov_b32_e32(v[1], 0),
+      v_cmp_eq_u32_e32(1, v[255]),  # vcc = (lane == 1)
+      v_cndmask_b32_e64(v[1], v[1], 1, VCC_LO),  # v1[lane1] = 1
+      v_cmp_ne_u32_e32(0, v[1]),  # vcc = (0 != v1)
+      v_cndmask_b32_e64(v[0], 0, 1, VCC_LO),  # v0 = vcc ? 1 : 0
+    ]
+    st = run_program(instructions, n_lanes=2)
+    self.assertEqual(st.vgpr[0][0], 0, "lane 0: 0 != 0 should be false")
+    self.assertEqual(st.vgpr[1][0], 1, "lane 1: 0 != 1 should be true")
+    self.assertEqual(st.vcc & 0x3, 0x2, "VCC should be 0b10")
+
+  def test_v_cmp_ne_u32_all_nonzero(self):
+    """V_CMP_NE_U32: all lanes have nonzero values."""
+    instructions = [
+      v_mov_b32_e32(v[1], 5),
+      v_cmp_ne_u32_e32(0, v[1]),
+    ]
+    st = run_program(instructions, n_lanes=4)
+    self.assertEqual(st.vcc & 0xf, 0xf, "All lanes should be != 0")
+
   def test_cmp_eq_u16_opsel_lo_lo(self):
     """V_CMP_EQ_U16 comparing lo halves."""
     instructions = [
@@ -448,6 +471,242 @@ class TestCmpFloat(unittest.TestCase):
     self.assertEqual((st.vcc >> 1) & 1, 0, "Lane 1: expected vcc=0 (2.0 < 1.0)")
 
 
+class TestVOP3VOPCModifiers(unittest.TestCase):
+  """Tests for VOP3 VOPC with abs/neg modifiers."""
+
+  def test_v_cmp_ge_f32_abs_both(self):
+    """v_cmp_ge_f32 with abs on both sources: abs(0.0) >= abs(-1.0) = false.
+
+    Regression test: int16 mod operation uses v_cmp_ge_f32 with abs modifiers.
+    """
+    instructions = [
+      v_mov_b32_e32(v[0], 0.0),
+      v_mov_b32_e32(v[1], -1.0),
+      # abs=0b11 means abs(src0) and abs(src1)
+      v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "abs(0.0) >= abs(-1.0) should be false")
+
+  def test_v_cmp_ge_f32_abs_negative_divisor(self):
+    """v_cmp_ge_f32 with abs: remainder check for negative divisor.
+
+    Tests the exact comparison used in int16 mod: abs(rem_f) >= abs(div_f).
+    For 1 % -1: rem_f = 0.0, div_f = -1.0, so abs(0.0) >= abs(-1.0) = false.
+    """
+    instructions = [
+      v_mov_b32_e32(v[0], 0.0),    # remainder as float
+      v_mov_b32_e32(v[1], -1.0),   # divisor as float
+      v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "abs(0.0) >= abs(-1.0) should be false")
+
+  def test_v_cmp_ge_f32_abs_small_remainder(self):
+    """v_cmp_ge_f32 with abs: abs(-0.5) >= abs(-3.0) = false."""
+    instructions = [
+      v_mov_b32_e32(v[0], -0.5),
+      v_mov_b32_e32(v[1], -3.0),
+      v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "abs(-0.5) >= abs(-3.0) should be false")
+
+  def test_v_cmp_ge_f32_abs_equal(self):
+    """v_cmp_ge_f32 with abs: abs(-1.0) >= abs(1.0) = true."""
+    instructions = [
+      v_mov_b32_e32(v[0], -1.0),
+      v_mov_b32_e32(v[1], 1.0),
+      v_cmp_ge_f32_e64(VCC_LO, v[0], v[1], abs=0b11),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "abs(-1.0) >= abs(1.0) should be true")
+
+
+class TestVOP3VOPC64Bit(unittest.TestCase):
+  """Tests for VOP3 VOPC with 64-bit operands."""
+
+  def test_v_cmp_lt_f64_basic(self):
+    """v_cmp_lt_f64: 0.0 < 1.0 = true."""
+    zero_f64 = f2i64(0.0)
+    one_f64 = f2i64(1.0)
+    instructions = [
+      s_mov_b32(s[0], zero_f64 & 0xffffffff),
+      s_mov_b32(s[1], zero_f64 >> 32),
+      s_mov_b32(s[2], one_f64 & 0xffffffff),
+      s_mov_b32(s[3], one_f64 >> 32),
+      v_cmp_lt_f64_e64(VCC_LO, s[0:1], s[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "0.0 < 1.0 should be true")
+
+  def test_v_cmp_lt_f64_negative(self):
+    """v_cmp_lt_f64: -1.0 < 0.0 = true."""
+    neg_one_f64 = f2i64(-1.0)
+    zero_f64 = f2i64(0.0)
+    instructions = [
+      s_mov_b32(s[0], neg_one_f64 & 0xffffffff),
+      s_mov_b32(s[1], neg_one_f64 >> 32),
+      s_mov_b32(s[2], zero_f64 & 0xffffffff),
+      s_mov_b32(s[3], zero_f64 >> 32),
+      v_cmp_lt_f64_e64(VCC_LO, s[0:1], s[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "-1.0 < 0.0 should be true")
+
+  def test_v_cmp_lt_i64_signed(self):
+    """v_cmp_lt_i64: 0 < -1 (signed) = false."""
+    instructions = [
+      s_mov_b32(s[0], 0),
+      s_mov_b32(s[1], 0),              # s[0:1] = 0
+      s_mov_b32(s[2], 0xffffffff),
+      s_mov_b32(s[3], 0xffffffff),     # s[2:3] = -1
+      v_cmp_lt_i64_e64(VCC_LO, s[0:1], s[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "0 < -1 (signed) should be false")
+
+  def test_v_cmp_lt_u64_unsigned(self):
+    """v_cmp_lt_u64: 0 < 0xFFFFFFFFFFFFFFFF (unsigned) = true."""
+    instructions = [
+      s_mov_b32(s[0], 0),
+      s_mov_b32(s[1], 0),              # s[0:1] = 0
+      s_mov_b32(s[2], 0xffffffff),
+      s_mov_b32(s[3], 0xffffffff),     # s[2:3] = max uint64
+      v_cmp_lt_u64_e64(VCC_LO, s[0:1], s[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "0 < max_uint64 should be true")
+
+
+class TestVOPCF64(unittest.TestCase):
+  """Tests for VOPC (E32 encoding) with 64-bit float operands. Regression test for f64 compare bug."""
+
+  def test_v_cmp_lt_f64_e32_true(self):
+    """v_cmp_lt_f64_e32: 2.0 < 3.0 = true."""
+    lo0, hi0 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
+    lo1, hi1 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
+    instructions = [
+      s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
+      s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
+      v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
+      v_cmp_lt_f64_e32(v[0:1], v[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "2.0 < 3.0 should be true")
+
+  def test_v_cmp_lt_f64_e32_false(self):
+    """v_cmp_lt_f64_e32: 3.0 < 2.0 = false."""
+    lo0, hi0 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
+    lo1, hi1 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
+    instructions = [
+      s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
+      s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
+      v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
+      v_cmp_lt_f64_e32(v[0:1], v[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "3.0 < 2.0 should be false")
+
+  def test_v_cmp_nlt_f64_e32_true(self):
+    """v_cmp_nlt_f64_e32: !(3.0 < 2.0) = true."""
+    lo0, hi0 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
+    lo1, hi1 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
+    instructions = [
+      s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
+      s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
+      v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
+      v_cmp_nlt_f64_e32(v[0:1], v[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(3.0 < 2.0) should be true")
+
+  def test_v_cmp_nlt_f64_e32_false(self):
+    """v_cmp_nlt_f64_e32: !(2.0 < 3.0) = false."""
+    lo0, hi0 = f2i64(2.0) & 0xffffffff, f2i64(2.0) >> 32
+    lo1, hi1 = f2i64(3.0) & 0xffffffff, f2i64(3.0) >> 32
+    instructions = [
+      s_mov_b32(s[0], lo0), s_mov_b32(s[1], hi0),
+      s_mov_b32(s[2], lo1), s_mov_b32(s[3], hi1),
+      v_mov_b32_e32(v[0], s[0]), v_mov_b32_e32(v[1], s[1]),
+      v_mov_b32_e32(v[2], s[2]), v_mov_b32_e32(v[3], s[3]),
+      v_cmp_nlt_f64_e32(v[0:1], v[2:3]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "!(2.0 < 3.0) should be false")
+
+
+class TestCmpxExec(unittest.TestCase):
+  """Tests for V_CMPX instructions that modify EXEC mask."""
+
+  def test_v_cmpx_ngt_f32_e64_all_true(self):
+    """V_CMPX_NGT_F32_E64: all lanes pass (literal <= all values)."""
+    # 131072.0 = 0x48000000
+    # All values > 131072, so !(131072 > val) = true for all
+    instructions = [
+      s_mov_b32(EXEC_LO, 0x7),  # 3 lanes active
+      v_mov_b32_e32(v[0], f2i(200000.0)),  # lane 0
+      v_cmp_eq_u32_e32(1, v[255]),
+      v_cndmask_b32_e64(v[1], v[0], f2i(300000.0), VCC_LO),  # lane 1
+      v_cmp_eq_u32_e32(2, v[255]),
+      v_cndmask_b32_e64(v[1], v[1], f2i(400000.0), VCC_LO),  # lane 2
+      # Now v[1] has: lane0=200000, lane1=300000, lane2=400000
+      # Compare: !(131072.0 > v[1]) i.e., 131072.0 <= v[1]
+      v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # All values > 131072, so all lanes should remain active
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x7, "All 3 lanes should remain active")
+
+  def test_v_cmpx_ngt_f32_e64_some_false(self):
+    """V_CMPX_NGT_F32_E64: some lanes fail (literal > some values)."""
+    instructions = [
+      s_mov_b32(EXEC_LO, 0x7),  # 3 lanes active
+      v_mov_b32_e32(v[0], f2i(100000.0)),  # lane 0: 131072 > 100000 = true, so !(true) = false
+      v_cmp_eq_u32_e32(1, v[255]),
+      v_cndmask_b32_e64(v[1], v[0], f2i(200000.0), VCC_LO),  # lane 1: 131072 > 200000 = false, so !(false) = true
+      v_cmp_eq_u32_e32(2, v[255]),
+      v_cndmask_b32_e64(v[1], v[1], f2i(150000.0), VCC_LO),  # lane 2: 131072 > 150000 = false, so !(false) = true
+      v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # lane 0: fail (100000 < 131072), lanes 1,2: pass
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x6, "Lanes 1,2 should be active, lane 0 inactive")
+
+  def test_v_cmpx_ngt_f32_e64_all_false(self):
+    """V_CMPX_NGT_F32_E64: all lanes fail (literal > all values)."""
+    instructions = [
+      s_mov_b32(EXEC_LO, 0x7),  # 3 lanes active
+      v_mov_b32_e32(v[0], f2i(100.0)),  # all lanes have 100.0
+      # 131072 > 100 = true, so !(true) = false for all
+      v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[0]),
+    ]
+    st = run_program(instructions, n_lanes=3)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x0, "All lanes should be inactive")
+
+  def test_v_cmpx_ngt_f32_e64_large_values(self):
+    """V_CMPX_NGT_F32_E64: test with values that trigger Payne-Hanek in sin().
+
+    This is a regression test for the sin(859240.0) bug.
+    Values 859240, 1000000, 100594688 should all pass !(131072 > val).
+    """
+    instructions = [
+      s_mov_b32(EXEC_LO, 0x7),  # 3 lanes active
+      v_mov_b32_e32(v[0], f2i(859240.0)),   # lane 0
+      v_cmp_eq_u32_e32(1, v[255]),
+      v_cndmask_b32_e64(v[1], v[0], f2i(1000000.0), VCC_LO),   # lane 1
+      v_cmp_eq_u32_e32(2, v[255]),
+      v_cndmask_b32_e64(v[1], v[1], f2i(100594688.0), VCC_LO), # lane 2
+      v_cmpx_ngt_f32_e64(EXEC_LO, f2i(131072.0), v[1]),
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # All values > 131072, so !(131072 > val) = true for all
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0x7, 0x7, "All 3 lanes should remain active")
+
+
 class TestVCCBehavior(unittest.TestCase):
   """Tests for VCC condition code behavior."""
 
@@ -472,5 +731,206 @@ class TestVCCBehavior(unittest.TestCase):
     self.assertEqual(st.vcc >> 16, 0x0000, "Lanes 16-31 should be false")
 
 
+class TestCmpNge(unittest.TestCase):
+  """Tests for V_CMP_NGE (not-greater-or-equal) with NaN semantics.
+
+  NGE = !(a >= b). With NaN inputs:
+  - If either input is NaN, a >= b is false, so !(false) = true
+  - This differs from a < b which returns false for NaN inputs
+  """
+
+  def test_v_cmp_nge_f32_normal_values(self):
+    """v_cmp_nge_f32: basic comparison with normal floats."""
+    instructions = [
+      v_mov_b32_e32(v[0], f2i(1.0)),
+      v_mov_b32_e32(v[1], f2i(2.0)),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(1.0 >= 2.0) = !(false) = true
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(1.0 >= 2.0) should be true")
+
+  def test_v_cmp_nge_f32_equal_values(self):
+    """v_cmp_nge_f32: equal values should return false."""
+    instructions = [
+      v_mov_b32_e32(v[0], f2i(1.0)),
+      v_mov_b32_e32(v[1], f2i(1.0)),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(1.0 >= 1.0) = !(true) = false
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "!(1.0 >= 1.0) should be false")
+
+  def test_v_cmp_nge_f32_greater_value(self):
+    """v_cmp_nge_f32: greater value should return false."""
+    instructions = [
+      v_mov_b32_e32(v[0], f2i(2.0)),
+      v_mov_b32_e32(v[1], f2i(1.0)),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(2.0 >= 1.0) = !(true) = false
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 0, "!(2.0 >= 1.0) should be false")
+
+  def test_v_cmp_nge_f32_neg_inf(self):
+    """v_cmp_nge_f32: -inf compared to normal value."""
+    neg_inf = 0xff800000  # -inf
+    instructions = [
+      s_mov_b32(s[0], neg_inf),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], f2i(1.0)),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(-inf >= 1.0) = !(false) = true
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(-inf >= 1.0) should be true")
+
+  def test_v_cmp_nge_f32_clears_inactive_vcc_bits(self):
+    """v_cmp_nge_f32 with partial EXEC clears inactive VCC bits (hardware behavior)."""
+    neg_inf = 0xff800000  # -inf
+    instructions = [
+      # Set VCC to all 1s first
+      s_mov_b32(VCC_LO, 0xFFFFFFFF),
+      # Set EXEC to only lane 0
+      s_mov_b32(EXEC_LO, 0x00000001),
+      # v0 = 1.0 for lane 0
+      v_mov_b32_e32(v[0], f2i(1.0)),
+      # Compare: !(-inf >= 1.0) = true for lane 0
+      v_cmp_nge_f32_e32(neg_inf, v[0]),
+    ]
+    st = run_program(instructions, n_lanes=16)
+    # Hardware clears inactive lane bits, only active lane results remain
+    # Lane 0 result = 1 (true), lanes 1-15 = 0 (cleared)
+    self.assertEqual(st.vcc, 0x00000001, "VCC should only have active lane results")
+
+  def test_v_cmp_nge_f32_nan_src0(self):
+    """v_cmp_nge_f32: NaN in src0 should return true (NaN >= x is false)."""
+    quiet_nan = 0x7fc00000
+    instructions = [
+      s_mov_b32(s[0], quiet_nan),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], f2i(1.0)),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(NaN >= 1.0) = !(false) = true
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(NaN >= 1.0) should be true")
+
+  def test_v_cmp_nge_f32_nan_src1(self):
+    """v_cmp_nge_f32: NaN in src1 should return true (x >= NaN is false)."""
+    quiet_nan = 0x7fc00000
+    instructions = [
+      s_mov_b32(s[0], quiet_nan),
+      v_mov_b32_e32(v[0], f2i(1.0)),
+      v_mov_b32_e32(v[1], s[0]),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(1.0 >= NaN) = !(false) = true
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(1.0 >= NaN) should be true")
+
+  def test_v_cmp_nge_f32_both_nan(self):
+    """v_cmp_nge_f32: both NaN should return true."""
+    quiet_nan = 0x7fc00000
+    instructions = [
+      s_mov_b32(s[0], quiet_nan),
+      v_mov_b32_e32(v[0], s[0]),
+      v_mov_b32_e32(v[1], s[0]),
+      v_cmp_nge_f32_e32(v[0], v[1]),  # !(NaN >= NaN) = !(false) = true
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vcc & 1, 1, "!(NaN >= NaN) should be true")
+
+
+class TestCmpxPartialWavefront(unittest.TestCase):
+  """Tests for V_CMPX with partial wavefronts (fewer than 32 active lanes).
+
+  Regression tests for bug where v_cmpx incorrectly set EXEC bits for inactive
+  lanes when the wavefront had fewer than 32 lanes. This caused garbage data
+  from uninitialized lanes to corrupt memory writes.
+  """
+
+  def test_v_cmpx_eq_u32_partial_wave_3_lanes(self):
+    """V_CMPX_EQ_U32 with 3 active lanes should only affect those 3 lanes.
+
+    With n_lanes=3, initial EXEC=0x7. After v_cmpx comparing lane_id == 1,
+    only lane 1 should pass, so EXEC should become 0x2 (not have bits 3-31 set).
+    """
+    instructions = [
+      v_cmpx_eq_u32_e32(1, v[255]),  # EXEC = lanes where lane_id == 1
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # Only lane 1 should be active (bit 1 set)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x2,
+                     "Only lane 1 should be active after v_cmpx_eq_u32 with 3 lanes")
+
+  def test_v_cmpx_eq_u32_partial_wave_5_lanes(self):
+    """V_CMPX_EQ_U32 with 5 active lanes."""
+    instructions = [
+      v_cmpx_eq_u32_e32(3, v[255]),  # EXEC = lanes where lane_id == 3
+    ]
+    st = run_program(instructions, n_lanes=5)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x8,
+                     "Only lane 3 should be active after v_cmpx_eq_u32 with 5 lanes")
+
+  def test_v_cmpx_lt_u32_partial_wave(self):
+    """V_CMPX_LT_U32 with partial wavefront."""
+    # VOPC: src0 < vsrc1, so we need v_cmpx_gt_u32 to get lane_id < 2
+    instructions = [
+      v_cmpx_gt_u32_e32(2, v[255]),  # EXEC = lanes where 2 > lane_id (i.e., lane_id < 2)
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # Lanes 0,1 should be active (bits 0,1 set = 0x3)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x3,
+                     "Only lanes 0,1 should be active after v_cmpx_gt_u32(2, lane_id) with 4 lanes")
+
+  def test_v_cmpx_ge_u32_partial_wave(self):
+    """V_CMPX_GE_U32 with partial wavefront."""
+    # VOPC: src0 >= vsrc1, so v_cmpx_le_u32(1, lane_id) gives lane_id >= 2? No.
+    # v_cmpx_le_u32(src0, vsrc1) = src0 <= vsrc1 = 1 <= lane_id
+    instructions = [
+      v_cmpx_le_u32_e32(2, v[255]),  # EXEC = lanes where 2 <= lane_id (i.e., lane_id >= 2)
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # Lanes 2,3 should be active (bits 2,3 set = 0xC)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0xC,
+                     "Only lanes 2,3 should be active after v_cmpx_le_u32(2, lane_id) with 4 lanes")
+
+  def test_v_cmpx_ne_u32_partial_wave_all_pass(self):
+    """V_CMPX_NE_U32 where all active lanes pass."""
+    instructions = [
+      v_cmpx_ne_u32_e32(99, v[255]),  # EXEC = lanes where lane_id != 99
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # All 3 lanes should remain active (bits 0,1,2 set = 0x7)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x7,
+                     "All 3 lanes should remain active when all pass")
+
+  def test_v_cmpx_eq_u32_partial_wave_none_pass(self):
+    """V_CMPX_EQ_U32 where no active lanes pass."""
+    instructions = [
+      v_cmpx_eq_u32_e32(99, v[255]),  # EXEC = lanes where lane_id == 99
+    ]
+    st = run_program(instructions, n_lanes=3)
+    # No lanes should be active
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x0,
+                     "No lanes should be active when none pass")
+
+  def test_v_cmpx_f32_partial_wave(self):
+    """V_CMPX_GT_F32 with partial wavefront - float comparison."""
+    instructions = [
+      v_cvt_f32_u32_e32(v[0], v[255]),  # v[0] = float(lane_id)
+      v_mov_b32_e32(v[1], f2i(0.5)),    # v[1] = 0.5
+      v_cmpx_gt_f32_e32(v[0], v[1]),    # EXEC = lanes where v[0] > 0.5
+    ]
+    st = run_program(instructions, n_lanes=4)
+    # Lanes 1,2,3 have values > 0.5, lane 0 has 0.0
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0xE,
+                     "Lanes 1,2,3 should be active (float > 0.5)")
+
+  def test_v_cmpx_e64_partial_wave(self):
+    """V_CMPX_EQ_U32_E64 (VOP3 encoding) with partial wavefront."""
+    instructions = [
+      v_cmpx_eq_u32_e64(EXEC_LO, v[255], 2),  # EXEC = lanes where lane_id == 2
+    ]
+    st = run_program(instructions, n_lanes=4)
+    self.assertEqual(st.sgpr[EXEC_LO.offset] & 0xFFFFFFFF, 0x4,
+                     "Only lane 2 should be active after v_cmpx_eq_u32_e64")
+
+
 if __name__ == '__main__':
   unittest.main()

extra/assembly/amd/test/hw/test_vopd.py

@@ -0,0 +1,161 @@
+"""Tests for VOPD instructions - dual-issue vector operations.
+
+VOPD executes two operations simultaneously. Key behavior:
+- Both ops read their sources BEFORE either writes (dual-issue semantics)
+- This means if X writes to a register that Y reads, Y sees the OLD value
+- Op X can use ops 0-15 (FMAC, MUL, ADD, MOV, etc.)
+- Op Y can use ops 0-18 (includes ADD_NC_U32, LSHLREV, AND)
+"""
+import unittest
+from extra.assembly.amd.test.hw.helpers import run_program, run_program_emu, run_program_hw, compare_wave_states, \
+  v, s, v_mov_b32_e32, s_mov_b32
+from extra.assembly.amd.autogen.rdna3.ins import VOPD, VOPD_LIT, VOPDOp
+
+class TestVOPDBasic(unittest.TestCase):
+  """Basic VOPD functionality tests."""
+
+  def test_vopd_dual_mov(self):
+    """VOPD with two MOV operations to different registers."""
+    instructions = [
+      v_mov_b32_e32(v[0], 0x12345678),
+      v_mov_b32_e32(v[1], 0xDEADBEEF),
+      # X: v[2] = v[0], Y: v[3] = v[1]
+      VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[1], v[0], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 0x12345678)
+    self.assertEqual(st.vgpr[0][3], 0xDEADBEEF)
+
+  def test_vopd_mov_and_add(self):
+    """VOPD with MOV (X) and ADD_NC_U32 (Y) - ADD_NC_U32 can only be Y op."""
+    instructions = [
+      v_mov_b32_e32(v[0], 10),
+      v_mov_b32_e32(v[1], 5),
+      # X: v[2] = 100 (literal), Y: v[3] = v[0] + v[1] = 15
+      VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[3], 100, v[0], v[0], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertEqual(st.vgpr[0][2], 100)
+    self.assertEqual(st.vgpr[0][3], 15)
+
+
+class TestVOPDReadBeforeWrite(unittest.TestCase):
+  """Tests for VOPD dual-issue read-before-write semantics.
+
+  In VOPD, both X and Y operations read their sources BEFORE either writes.
+  This is critical when X's destination is Y's source.
+  """
+
+  def test_vopd_x_writes_y_reads_same_reg(self):
+    """VOPD where X writes to a register that Y reads.
+
+    X: v[2] = 0 (overwrites v[2])
+    Y: v[1] = v[2] + v[0]  (srcy0=v[2], vsrcy1=v[0])
+
+    If reads happen before writes: v[1] = OLD_v[2] + v[0] = 0xFFFFFFFF + 1 = 0
+    If writes happen before reads: v[1] = 0 + v[0] = 0 + 1 = 1
+
+    Hardware does reads-before-writes, so v[1] should be 0.
+    """
+    instructions = [
+      v_mov_b32_e32(v[0], 1),          # v[0] = 1
+      v_mov_b32_e32(v[1], 0x99999999), # v[1] = placeholder (will be overwritten)
+      v_mov_b32_e32(v[2], 0xFFFFFFFF), # v[2] = 0xFFFFFFFF
+      # X: v[2] = 0 (literal), srcx0=0, vsrcx1=v[0] (unused for MOV)
+      # Y: v[1] = srcy0 + vsrcy1 = v[2] + v[0] (should read OLD v[2] = 0xFFFFFFFF)
+      # vdsty encoding: (vdsty << 1) | ((vdstx & 1) ^ 1) where vdsty field = 0, vdstx = v[2]
+      # So vdsty_reg = (0 << 1) | ((2 & 1) ^ 1) = 0 | 1 = 1 = v[1]
+      VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 0, v[2], v[0], v[0]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # X should have written 0 to v[2]
+    self.assertEqual(st.vgpr[0][2], 0, "X should write 0 to v[2]")
+    # Y should have read OLD v[2] (0xFFFFFFFF) and added v[0] (1)
+    # 0xFFFFFFFF + 1 = 0 (wrap around)
+    self.assertEqual(st.vgpr[0][1], 0, "Y should read OLD v[2]=0xFFFFFFFF, compute 0xFFFFFFFF+1=0")
+
+  def test_vopd_x_writes_y_reads_same_reg_v2(self):
+    """VOPD where X writes to a register that Y reads - cleaner test case.
+
+    X: v[2] = 0 (MOV)
+    Y: v[1] = v[2] + v[2] (ADD_NC_U32 with both sources from v[2])
+
+    If reads happen before writes: v[1] = OLD_v[2] + OLD_v[2] = 100 + 100 = 200
+    If writes happen before reads: v[1] = 0 + 0 = 0
+
+    Hardware does reads-before-writes, so v[1] should be 200.
+    """
+    instructions = [
+      v_mov_b32_e32(v[0], 0x88888888), # v[0] = unused placeholder
+      v_mov_b32_e32(v[1], 0x99999999), # v[1] = placeholder (will be overwritten)
+      v_mov_b32_e32(v[2], 100),        # v[2] = 100
+      # X: v[2] = 0 (literal)
+      # Y: v[1] = srcy0 + vsrcy1 = v[2] + v[2] (should read OLD v[2] = 100)
+      VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 0, v[2], v[0], v[2]),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    # X should have written 0 to v[2]
+    self.assertEqual(st.vgpr[0][2], 0, "X should write 0 to v[2]")
+    # Y should have read OLD v[2] (100) twice and added them
+    self.assertEqual(st.vgpr[0][1], 200, "Y should read OLD v[2]=100 twice, compute 100+100=200")
+
+
+class TestVOPDLiterals(unittest.TestCase):
+  """Tests for VOPD instructions that use SIMM32 literals (FMAAK, FMAMK)."""
+
+  def test_vopd_fmaak_f32(self):
+    """VOPD V_DUAL_FMAAK_F32: D = S0 * S1 + SIMM32 (literal addend).
+
+    Tests that the 32-bit literal (SIMM32) is correctly passed to the instruction.
+    fma(2.0, 3.0, 10.0) = 2*3 + 10 = 16.0
+    """
+    from extra.assembly.amd.test.hw.helpers import f2i, i2f
+    instructions = [
+      v_mov_b32_e32(v[0], f2i(2.0)),  # v[0] = 2.0
+      v_mov_b32_e32(v[1], f2i(3.0)),  # v[1] = 3.0
+      # VOPD args: opx, opy, vdstx, vdsty, srcx0, srcy0, vsrcx1, vsrcy1
+      # X: v[2] = fma(srcx0, vsrcx1, SIMM32) = v[0]*v[1]+10.0 = 2*3+10 = 16
+      # Y: v[3] = srcy0 (MOV) = v[0] = 2.0
+      VOPD_LIT(VOPDOp.V_DUAL_FMAAK_F32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[0], v[1], v[0], literal=f2i(10.0)),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertAlmostEqual(i2f(st.vgpr[0][2]), 16.0, places=5, msg="fma(2.0, 3.0, 10.0) should be 16.0")
+
+  def test_vopd_fmamk_f32(self):
+    """VOPD V_DUAL_FMAMK_F32: D = S0 * SIMM32 + S1 (literal multiplier).
+
+    Tests that the 32-bit literal (SIMM32) is correctly used as the multiplier.
+    fma(2.0, 5.0, 3.0) = 2*5 + 3 = 13.0
+    """
+    from extra.assembly.amd.test.hw.helpers import f2i, i2f
+    instructions = [
+      v_mov_b32_e32(v[0], f2i(2.0)),  # v[0] = 2.0
+      v_mov_b32_e32(v[1], f2i(3.0)),  # v[1] = 3.0
+      # X: v[2] = fma(srcx0, SIMM32, vsrcx1) = v[0]*5.0+v[1] = 2*5+3 = 13
+      # Y: v[3] = srcy0 (MOV) = v[0] = 2.0
+      VOPD_LIT(VOPDOp.V_DUAL_FMAMK_F32, VOPDOp.V_DUAL_MOV_B32, v[2], v[3], v[0], v[0], v[1], v[0], literal=f2i(5.0)),
+    ]
+    st = run_program(instructions, n_lanes=1)
+    self.assertAlmostEqual(i2f(st.vgpr[0][2]), 13.0, places=5, msg="fma(2.0, 5.0, 3.0) should be 13.0")
+
+
+class TestVOPDMultilane(unittest.TestCase):
+  """Tests for VOPD with multiple lanes."""
+
+  def test_vopd_multilane_mov_add(self):
+    """VOPD MOV and ADD with multiple active lanes - no register conflict."""
+    instructions = [
+      v_mov_b32_e32(v[0], 5),
+      v_mov_b32_e32(v[1], 10),
+      # X: v[2] = 100 (constant), Y: v[1] = v[0] + v[1] = 5 + 10 = 15
+      # vdsty_reg = (vdsty << 1) | ((vdstx.offset & 1) ^ 1) = (0 << 1) | ((258 & 1) ^ 1) = 0 | 1 = 1
+      VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_ADD_NC_U32, v[2], v[0], 100, v[0], v[2], v[1]),
+    ]
+    st = run_program(instructions, n_lanes=4)
+    for lane in range(4):
+      self.assertEqual(st.vgpr[lane][2], 100, f"Lane {lane}: v[2] should be 100")
+      self.assertEqual(st.vgpr[lane][1], 15, f"Lane {lane}: v[1] should be 15 (5+10)")
+
+
+if __name__ == '__main__':
+  unittest.main()

extra/assembly/amd/test/test_compare_emulators.py

@@ -1,18 +1,14 @@
 # Test to compare Python and Rust RDNA3 emulators by running real tinygrad kernels
-import unittest, ctypes, os
+import unittest, ctypes
 from dataclasses import dataclass
-from pathlib import Path
 
-# Set environment before any tinygrad imports to use MOCKGPU
-# This allows generating AMD GPU kernels without requiring real hardware
-os.environ["AMD"] = "1"
-os.environ["MOCKGPU"] = "1"
-os.environ["PYTHON_REMU"] = "1"
-
-from extra.assembly.amd.emu import WaveState, decode_program, WAVE_SIZE, set_valid_mem_ranges, LDSMem
+from extra.assembly.amd.emu import WaveState, decode_program, WAVE_SIZE, VCC_LO, EXEC_LO, SCC
+from extra.assembly.amd import decode_inst
 from extra.assembly.amd.test.helpers import KernelInfo
 from extra.assembly.amd.test.bench_emu import REMU_PATH
 
+def set_valid_mem_ranges(ranges): pass  # emu2 doesn't need this
+
 def _is_f32_nan(bits: int) -> bool:
   """Check if 32-bit value is a NaN (exponent all 1s, mantissa non-zero)."""
   return (bits & 0x7f800000) == 0x7f800000 and (bits & 0x007fffff) != 0
@@ -91,33 +87,61 @@ class PythonEmulator:
   def __init__(self):
     self.state: WaveState | None = None
     self.program: dict | None = None
+    self.vmem_buf = None
+    self.lds_buf = None
+    self.kernel_buf = None  # Keep kernel bytes alive
+    self.lib_addr = 0  # Base address of kernel code
 
   def create(self, kernel: bytes, n_lanes: int):
-    self.program = decode_program(kernel)
-    self.state = WaveState(LDSMem(bytearray(65536)), n_lanes)
-    self.state.exec_mask = (1 << n_lanes) - 1
+    import ctypes
+    from tinygrad.device import Buffer, BufferSpec
+    from tinygrad.dtype import dtypes
+    # Store kernel in a ctypes buffer so generic instructions can read from vmem at actual PC address
+    self.kernel_buf = (ctypes.c_char * len(kernel)).from_buffer_copy(kernel)
+    self.lib_addr = ctypes.addressof(self.kernel_buf)
+    # Remap program dict to use actual addresses (like run_asm does)
+    program_raw = decode_program(kernel)
+    self.program = {self.lib_addr + offset: val for offset, val in program_raw.items()}
+    self.state = WaveState(n_lanes)
+    self.state.pc = self.lib_addr  # Set PC to code base address
+    self.vmem_buf = Buffer('CPU', 1 << 40, dtypes.uint32, options=BufferSpec(external_ptr=0)).ensure_allocated()
+    self.lds_buf = Buffer('CPU', 65536 // 4, dtypes.uint32).ensure_allocated()
 
   def step(self) -> int:
+    import ctypes
     assert self.program is not None and self.state is not None
-    return self.program[self.state.pc]._dispatch(self.state, self.program[self.state.pc])
+    pc = self.state.pc
+    if pc == 0xFFFFFFFFFFFFFFFF or pc not in self.program: return -1
+    name, fxn, globals_list, _runner = self.program[pc]
+    if fxn is None: return 1  # unsupported instruction
+    buf_addrs = {0: self.state.sgpr_buf._buf.va_addr, 1: self.state.vgpr_buf._buf.va_addr,
+                 2: self.vmem_buf._buf.va_addr, 3: self.lds_buf._buf.va_addr}
+    # Direct ctypes call - bypasses HCQ overhead
+    fxn(*[ctypes.c_uint64(buf_addrs[g]) for g in globals_list], ctypes.c_int32(0))
+    return -1 if self.state.pc == 0xFFFFFFFFFFFFFFFF else 0
+
   def set_sgpr(self, idx: int, val: int):
     assert self.state is not None
-    self.state.sgpr[idx] = val & 0xffffffff
+    self.state._write_sgpr(idx, val)
   def set_vgpr(self, lane: int, idx: int, val: int):
     assert self.state is not None
-    self.state.vgpr[lane][idx] = val & 0xffffffff
+    self.state._write_vgpr(idx, lane, val)
 
   def get_snapshot(self) -> StateSnapshot:
     assert self.state is not None
-    return StateSnapshot(pc=self.state.pc, scc=self.state.scc, vcc=self.state.vcc & 0xffffffff,
-                         exec_mask=self.state.exec_mask & 0xffffffff, sgpr=list(self.state.sgpr),
-                         vgpr=[list(self.state.vgpr[i]) for i in range(WAVE_SIZE)])
+    sgpr = [self.state._read_sgpr(i) for i in range(128)]
+    vgpr = [[self.state._read_vgpr(reg, lane) for reg in range(256)] for lane in range(WAVE_SIZE)]
+    # Convert actual PC address to word offset for comparison with Rust emulator
+    pc_offset = (self.state.pc - self.lib_addr) // 4 if self.state.pc != 0xFFFFFFFFFFFFFFFF else 0xFFFFFFFFFFFFFFFF
+    return StateSnapshot(pc=pc_offset, scc=self.state._read_sgpr(SCC.offset), vcc=sgpr[VCC_LO.offset],
+                         exec_mask=sgpr[EXEC_LO.offset], sgpr=sgpr, vgpr=vgpr)
 
 def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: tuple[int, int, int],
-                      program, max_steps: int, debug: bool, trace_len: int, kernel_idx: int = 0,
-                      max_workgroups: int = 8) -> tuple[bool, str, int]:
+                      local_size: tuple[int, int, int], program, max_steps: int, debug: bool, trace_len: int,
+                      kernel_idx: int = 0, max_workgroups: int = 8) -> tuple[bool, str, int]:
   """Run a single kernel through both emulators. Returns (success, message, total_steps)."""
   gx, gy, gz = global_size
+  lx, ly, lz = local_size
   total_steps = 0
   wg_count = 0
 
@@ -140,28 +164,52 @@ def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: t
           emu.set_sgpr(13, gidx)
           emu.set_sgpr(14, gidy)
           emu.set_sgpr(15, gidz)
+          # Initialize v[0] with packed workitem IDs for each lane
+          for lane in range(n_lanes):
+            tid = lane
+            z, y, x = tid // (lx * ly), (tid // lx) % ly, tid % lx
+            emu.set_vgpr(lane, 0, (z << 20) | (y << 10) | x)
 
         step = 0
         trace: list[tuple[int, int, str, StateSnapshot, StateSnapshot]] = []
+        prev_sync_after = False  # Track if previous instruction had known Rust bugs
         try:
           while step < max_steps:
             rust_before = rust.get_snapshot()
             python_before = python.get_snapshot()
 
-            inst = program.get(python_before.pc)
-            inst_str = inst.disasm() if inst else f"unknown at PC={python_before.pc}"
+            inst_info = python.program.get(python.lib_addr + python_before.pc * 4)  # Convert word offset to actual address
+            inst_hex_name = inst_info[0] if inst_info else f"unknown at PC={python_before.pc}"
+            # Decode the instruction to get mnemonic for sync_after checks
+            try:
+              # Format is mnemonic_hexbytes, e.g. v_exp_f32_e32_014b027e -> hex is 014b027e
+              parts = inst_hex_name.rsplit('_', 1)
+              inst_bytes_hex = parts[1] if len(parts) == 2 else ""
+              inst_bytes = bytes.fromhex(inst_bytes_hex) if inst_bytes_hex else b''
+              decoded = decode_inst(inst_bytes) if inst_bytes else None
+              inst_mnemonic = repr(decoded).split('(')[0] if decoded else ""
+            except:
+              inst_mnemonic = ""
+            # For generic instructions, use function name for sync_after check
+            if not inst_mnemonic: inst_mnemonic = inst_hex_name
+            inst_str = inst_hex_name
             trace.append((step, python_before.pc, inst_str, rust_before, python_before))
             if len(trace) > trace_len: trace.pop(0)
 
             if debug: print(f"K{kernel_idx} WG({gidx},{gidy},{gidz}) Step {step}: PC={python_before.pc}, inst={inst_str}")
 
-            # Instructions with known Rust emulator bugs - sync Python to Rust after execution
+            # Instructions with known Rust emulator bugs or precision differences - sync Python to Rust after execution
             # v_div_scale/v_div_fixup: Rust has different VCC handling
             # v_cvt_f16_f32: Rust clears high 16 bits, but hardware (and Python) preserves them
             # s_add_i32/s_sub_i32: Rust has incorrect SCC overflow detection
-            sync_after = any(x in inst_str for x in ('v_div_scale_f32', 'v_div_scale_f64', 'v_div_fixup_f32', 'v_div_fixup_f64',
-                                                      'v_cvt_f16_f32', 's_add_i32', 's_sub_i32'))
-            diffs = rust_before.diff(python_before, n_lanes)
+            # v_exp_f32/v_log_f32/v_ldexp_f32: precision differences in transcendental functions
+            # s_delay_alu: Rust handles differently
+            # v_add_co_ci_u32/v_sub_co_ci_u32/v_subrev_co_ci_u32: Rust preserves inactive VCC bits, but hardware clears all bits
+            sync_after = any(x in inst_mnemonic.lower() for x in ('v_div_scale', 'v_div_fixup', 'v_cvt_f16_f32', 's_add_i32', 's_sub_i32',
+                                                                   'v_exp_f32', 'v_log_f32', 'v_ldexp_f32', 's_delay_alu',
+                                                                   'v_add_co_ci_u32', 'v_sub_co_ci_u32', 'v_subrev_co_ci_u32'))
+            # Skip comparison if previous instruction had known Rust bugs (states were synced but may still differ slightly)
+            diffs = rust_before.diff(python_before, n_lanes) if not prev_sync_after else []
             if diffs:
               trace_lines = []
               for idx, (s, pc, d, rb, pb) in enumerate(trace):
@@ -200,7 +248,12 @@ def run_single_kernel(kernel: bytes, n_lanes: int, args_ptr: int, global_size: t
               for lane in range(n_lanes):
                 for i in range(256): python.set_vgpr(lane, i, rust_after.vgpr[lane][i])
               assert python.state is not None
-              python.state.pc, python.state.scc, python.state.vcc, python.state.exec_mask = rust_after.pc, rust_after.scc, rust_after.vcc, rust_after.exec_mask
+              # Convert Rust's word-based PC to Python's actual address
+              python.state.pc = python.lib_addr + rust_after.pc * 4
+              python.state._write_sgpr(SCC.offset, rust_after.scc)
+              python.state._write_sgpr(VCC_LO.offset, rust_after.vcc)
+              python.state._write_sgpr(EXEC_LO.offset, rust_after.exec_mask)
+            prev_sync_after = sync_after
 
             if rust_result == -1:
               total_steps += step + 1
@@ -254,7 +307,7 @@ def compare_emulators_multi_kernel(kernels: list[KernelInfo], buf_pool: dict[int
 
     ok, msg, steps = run_single_kernel(
       kernel.code, min(n_lanes, 32), args_ptr, kernel.global_size,
-      program, max_steps, debug, trace_len, ki
+      kernel.local_size, program, max_steps, debug, trace_len, ki
     )
     total_steps += steps
     if not ok:
@@ -281,7 +334,8 @@ def compare_emulators_with_memory(kernel: bytes, n_lanes: int, buf_sizes: list,
   set_valid_mem_ranges(ranges)
 
   program = decode_program(kernel)
-  ok, msg, _ = run_single_kernel(kernel, n_lanes, args_ptr, global_size, program, max_steps, debug, trace_len)
+  # Legacy wrapper assumes local_size = (n_lanes, 1, 1)
+  ok, msg, _ = run_single_kernel(kernel, n_lanes, args_ptr, global_size, (n_lanes, 1, 1), program, max_steps, debug, trace_len)
   return ok, msg
 
 def get_kernels_from_tinygrad(op_fn) -> tuple[list[KernelInfo], dict[int, int], dict[int, bytes]]:
@@ -324,6 +378,7 @@ def get_kernels_from_tinygrad(op_fn) -> tuple[list[KernelInfo], dict[int, int],
               buf_sizes.append(b.nbytes)
             kernels.append(KernelInfo(
               code=bytes(sec.content),
+              src=lowered.prg.p.src,
               global_size=tuple(lowered.prg.p.global_size),
               local_size=tuple(lowered.prg.p.local_size),
               buf_idxs=buf_idxs,
@@ -386,6 +441,7 @@ class TestTinygradKernels(unittest.TestCase):
     from tinygrad import dtypes
     self._test_kernel(lambda T: T.empty(4, 4)[T.arange(4).cast(dtypes.int64), :])
   def test_gelu(self): self._test_kernel(lambda T: T.empty(32, 32).gelu())
+  def test_exp(self): self._test_kernel(lambda T: T.empty(1024).exp())
   def test_cross_entropy(self):
     import numpy as np
     np.random.seed(0)
@@ -397,5 +453,55 @@ class TestTinygradKernels(unittest.TestCase):
     from tinygrad import dtypes
     self._test_kernel(lambda T: T([2.0], dtype=dtypes.float64).sin())
 
+  def test_sin_large_f32(self):
+    """Test sin with large values that trigger Payne-Hanek range reduction."""
+    # Values around 859240 trigger the Payne-Hanek algorithm
+    # This tests the integer multiply-high instructions used in range reduction
+    self._test_kernel(lambda T: T([859240.0, 1000000.0, 100594688.0]).sin())
+
+  def test_clip_zero_one(self):
+    """Test clip(0, 1) - regression for binary_crossentropy failure."""
+    import numpy as np
+    np.random.seed(0)
+    x_np = np.random.uniform(-2, 2, (32, 10)).astype(np.float32).tolist()
+    self._test_kernel(lambda T: T(x_np).clip(0, 1))
+
+  def test_mod_int64(self):
+    """Test int64 modulo, especially edge cases like 1 % -1."""
+    from tinygrad import dtypes
+    self._test_kernel(lambda T: T([1, 10, -10, 7], dtype=dtypes.int64) % T([-1, 3, 3, -3], dtype=dtypes.int64))
+
+  def test_expand_flatten_sum(self):
+    """Test flatten of expanded tensor followed by sum.
+
+    Bug: flatten() of an expanded tensor produces wrong results for certain sizes.
+    Sizes that are multiples of 32 work (32, 48, 64), but sizes like 33, 49, 50 fail.
+    This breaks masked_select and nonzero operations.
+    """
+    import numpy as np
+    np.random.seed(0)
+    x_np = np.random.uniform(-2, 2, (33,)).astype(np.float32)
+    self._test_kernel(lambda T: (T(x_np.tolist()) > 0.5).unsqueeze(-1).expand(33, 3).flatten().sum())
+
+  @unittest.skip("slow and broken with AMD_LLVM=1")
+  def test_nonzero(self):
+    """Test nonzero operation - counts and gathers indices of non-zero elements."""
+    import numpy as np
+    np.random.seed(42)
+    x_np = np.random.rand(10, 5, 3).astype(np.float32)
+    self._test_kernel(lambda T: (T(x_np.tolist()) > 0.5).nonzero())
+
+  @unittest.skip("Precision differences in v_exp/v_log accumulate across kernels, causing memory divergence")
+  def test_softmax_argmax_fused(self):
+    """Test fused softmax+argmax - tracks exp2 precision issue.
+
+    The fused kernel recomputes softmax inline and Python emulator's exp2 polynomial
+    has up to 1 ULP error vs native exp2f, causing accumulated differences.
+    """
+    import torch
+    torch.manual_seed(0)
+    x_np = torch.rand(4, 10).numpy()
+    self._test_kernel(lambda T: T(x_np.tolist()).softmax(1).argmax())
+
 if __name__ == "__main__":
   unittest.main()

extra/assembly/amd/test/test_custom_kernel.py

@@ -34,15 +34,44 @@ def custom_add_one(A:UOp, arch:str) -> UOp:
   sink = UOp.sink(A.base, threads, arg=KernelInfo(name:=f"custom_add_one_{A.size}", estimates=Estimates(ops=A.size, mem=A.size*4*2)))
   return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=(*sink.src, sink)), *assemble_insts(insts, name, arch)))
 
+def custom_add_var(A:UOp, B:UOp, arch:str) -> UOp:
+  A,B = A.flatten(), B.flatten()
+  assert A.dtype.base == dtypes.uint32, f"buffer dtype must be uint32, got {A.dtype}"
+  threads = UOp.special(A.size, "lidx0")
+  var = UOp.variable("var", 0, 10)
+  insts = [
+    s_load_b128(s[4:7], s[0:1]),
+    s_load_b32(s[8], s[0:1], offset=0x10), # all threads load the same variable
+    s_waitcnt(lgkmcnt=0),
+    v_lshlrev_b32_e32(v[0], 2, v[0]), # element offset, different per thread
+    global_load_b32(v[1], v[0], saddr=s[6:7]),
+    s_waitcnt(vmcnt=0),
+    v_add_nc_u32_e32(v[1], s[8], v[1]),
+    global_store_b32(addr=v[0], data=v[1], saddr=s[4:5]),
+    s_endpgm(),
+  ]
+  sink = UOp.sink(A.base, B.base, var, threads, arg=KernelInfo(name:=f"custom_add_one_{A.size}"))
+  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg="AMD"), UOp(Ops.LINEAR, src=(*sink.src, sink)),
+                               *assemble_insts(insts, name, arch, kernarg_size=16)))
+
 class TestCustomKernel(unittest.TestCase):
   def test_simple(self):
     a = Tensor.full((16, 16), 1.).contiguous().realize()
-    a = Tensor.custom_kernel(a, fxn=functools.partial(custom_add_one, arch=Device[Device.DEFAULT].arch))[0]
+    a = Tensor.custom_kernel(a, fxn=functools.partial(custom_add_one, arch=Device[Device.DEFAULT].renderer.arch))[0]
     ei = a.schedule()[-1].lower()
     self.assertEqual(ei.prg.estimates.ops, a.numel())
     self.assertEqual(ei.prg.estimates.mem, a.nbytes()*2)
     ei.run()
     self.assertTrue((a.numpy() == 2.).all())
 
+  def test_variable(self):
+    b = Tensor.full((16, 16), 1, dtype=dtypes.uint32).contiguous().realize()
+    a = Tensor.zeros_like(b).contiguous().realize()
+    a = Tensor.custom_kernel(a, b, fxn=functools.partial(custom_add_var, arch=Device[Device.DEFAULT].renderer.arch))[0]
+    ei = a.schedule()[-1].lower()
+    for i in range(4):
+      ei.run({"var":i})
+      self.assertTrue((a.numpy() == 1+i).all())
+
 if __name__ == "__main__":
   unittest.main()

extra/assembly/amd/test/test_emu2_pcode.py

@@ -0,0 +1,332 @@
+"""Tests for the pcode parser."""
+import unittest
+from collections import defaultdict
+from tinygrad.helpers import DEBUG
+from tinygrad.dtype import dtypes
+from tinygrad.uop.ops import UOp, Ops
+from extra.assembly.amd.emu import parse_pcode
+from extra.assembly.amd.pcode import parse_expr
+from extra.assembly.amd.autogen.rdna3.str_pcode import PCODE
+from extra.assembly.amd.autogen.rdna3.enum import VOP1Op, VOP2Op, VOP3Op, SOP1Op, SOP2Op, DSOp
+
+def _srcs():
+  """Create minimal source variables for pcode parsing."""
+  u32 = lambda v=0: UOp.const(dtypes.uint32, v)
+  return {'S0': u32(), 'S1': u32(), 'S2': u32(), 'SCC': u32(), 'VCC': UOp.const(dtypes.uint64, 0), 'laneId': u32()}
+
+class TestBasicParsing(unittest.TestCase):
+  """Test basic pcode parsing for common instruction patterns."""
+
+  def test_v_add_f32(self):
+    """Test parsing V_ADD_F32 pcode."""
+    _, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_F32_E32], _srcs())
+    self.assertEqual(len(assigns), 1)
+    dest, _ = assigns[0]
+    self.assertTrue(dest.startswith('D0'))
+
+  def test_v_lshlrev_b32(self):
+    """Test parsing V_LSHLREV_B32 pcode."""
+    _, assigns = parse_pcode(PCODE[VOP2Op.V_LSHLREV_B32_E32], _srcs())
+    self.assertEqual(len(assigns), 1)
+
+  def test_s_cselect_b32(self):
+    """Test parsing S_CSELECT_B32 pcode with ternary."""
+    _, assigns = parse_pcode(PCODE[SOP2Op.S_CSELECT_B32], _srcs())
+    self.assertEqual(len(assigns), 1)
+
+  def test_v_add_co_ci_u32(self):
+    """Test parsing V_ADD_CO_CI_U32 with carry."""
+    _, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_CO_CI_U32_E32], _srcs())
+    self.assertGreaterEqual(len(assigns), 1)
+
+class TestWithSources(unittest.TestCase):
+  """Test pcode parsing with actual source operand values."""
+
+  def test_v_add_f32_with_sources(self):
+    """Test V_ADD_F32 with actual float constants."""
+    s0 = UOp.const(dtypes.uint32, 0x3f800000)  # 1.0f
+    s1 = UOp.const(dtypes.uint32, 0x40000000)  # 2.0f
+    _, assigns = parse_pcode(PCODE[VOP2Op.V_ADD_F32_E32], {'S0': s0, 'S1': s1})
+    self.assertEqual(len(assigns), 1)
+    dest, val = assigns[0]
+    self.assertTrue(dest.startswith('D0'))
+    # Result should be an ADD operation
+    self.assertEqual(val.op, Ops.ADD)
+
+  def test_v_mul_f32_with_sources(self):
+    """Test V_MUL_F32 with actual float constants."""
+    s0 = UOp.const(dtypes.uint32, 0x40000000)  # 2.0f
+    s1 = UOp.const(dtypes.uint32, 0x40400000)  # 3.0f
+    _, assigns = parse_pcode(PCODE[VOP2Op.V_MUL_F32_E32], {'S0': s0, 'S1': s1})
+    self.assertEqual(len(assigns), 1)
+    dest, val = assigns[0]
+    self.assertEqual(val.op, Ops.MUL)
+
+class TestParseExpr(unittest.TestCase):
+  """Test the parse_expr function directly."""
+
+  def test_integer_literals(self):
+    """Test parsing integer literals."""
+    self.assertEqual(parse_expr('0', {}).arg, 0)
+    self.assertEqual(parse_expr('42', {}).arg, 42)
+    self.assertEqual(parse_expr('42U', {}).arg, 42)
+
+  def test_negative_integers(self):
+    """Test parsing negative integer literals."""
+    result = parse_expr('-1', {})
+    self.assertEqual(result.arg, -1)
+    self.assertEqual(result.dtype, dtypes.int)
+
+  def test_float_literals(self):
+    """Test parsing float literals."""
+    result = parse_expr('1.0F', {})
+    self.assertEqual(result.arg, 1.0)
+    self.assertEqual(result.dtype, dtypes.float32)
+
+  def test_hex_literals(self):
+    """Test parsing hex literals."""
+    result = parse_expr('0xFF', {})
+    self.assertEqual(result.arg, 255)
+
+  def test_variable_lookup(self):
+    """Test variable lookup in parse_expr."""
+    vars = {'x': UOp.const(dtypes.uint32, 42)}
+    result = parse_expr('x', vars)
+    self.assertEqual(result.arg, 42)
+
+  def test_binary_ops(self):
+    """Test parsing binary operations."""
+    vars = {'a': UOp.const(dtypes.uint32, 10), 'b': UOp.const(dtypes.uint32, 5)}
+
+    # Addition
+    result = parse_expr('a + b', vars)
+    self.assertEqual(result.op, Ops.ADD)
+
+    # Subtraction with constant folding
+    result = parse_expr('10 - 5', {})
+    self.assertEqual(result.op, Ops.CONST)
+    self.assertEqual(result.arg, 5)
+
+  def test_ternary(self):
+    """Test parsing ternary expressions."""
+    vars = {'cond': UOp.const(dtypes.bool, True), 'a': UOp.const(dtypes.uint32, 1), 'b': UOp.const(dtypes.uint32, 0)}
+    result = parse_expr('cond ? a : b', vars)
+    self.assertEqual(result.op, Ops.WHERE)
+
+class TestForLoopParsing(unittest.TestCase):
+  """Test for loop parsing (CLZ/CTZ patterns)."""
+
+  def test_clz_pcode_exists(self):
+    """Verify CLZ pcode is available."""
+    pcode = PCODE.get(VOP1Op.V_CLZ_I32_U32_E32)
+    self.assertIsNotNone(pcode)
+    self.assertIn('for', pcode.lower())
+
+  def test_clz_parsing(self):
+    """Test CLZ pcode parsing produces correct structure."""
+    pcode = PCODE[VOP1Op.V_CLZ_I32_U32_E32]
+    S0 = UOp.const(dtypes.uint32, 0xFFFFFFFF)  # All ones - CLZ should be 0
+    vars, assigns = parse_pcode(pcode, {'S0': S0})
+
+    self.assertEqual(len(assigns), 1)
+    dest, val = assigns[0]
+    self.assertTrue(dest.startswith('D0'))
+    # Result should be a nested WHERE structure
+    self.assertEqual(val.op, Ops.WHERE)
+
+  def test_clz_with_zero(self):
+    """Test CLZ with input 0 - should return -1."""
+    pcode = PCODE[VOP1Op.V_CLZ_I32_U32_E32]
+    S0 = UOp.const(dtypes.uint32, 0)
+    vars, assigns = parse_pcode(pcode, {'S0': S0})
+
+    # Check that the innermost value (default) is -1 (may be wrapped in CAST)
+    val = assigns[0][1]
+    # Traverse to innermost WHERE
+    while val.op == Ops.WHERE:
+      val = val.src[2]  # false branch
+    # Unwrap CAST if present
+    while val.op == Ops.CAST:
+      val = val.src[0]
+    self.assertEqual(val.arg, -1)
+
+  def test_ctz_parsing(self):
+    """Test CTZ pcode parsing."""
+    pcode = PCODE.get(VOP1Op.V_CTZ_I32_B32_E32)
+    if pcode is None:
+      self.skipTest("V_CTZ_I32_B32_E32 pcode not available")
+
+    S0 = UOp.const(dtypes.uint32, 1)  # LSB set - CTZ should be 0
+    vars, assigns = parse_pcode(pcode, {'S0': S0})
+    self.assertEqual(len(assigns), 1)
+
+class TestDSPcodePatterns(unittest.TestCase):
+  """Test DS instruction pcode patterns."""
+
+  def test_ds_load_b32_pcode(self):
+    """Test DS_LOAD_B32 pcode is parseable."""
+    pcode = PCODE.get(DSOp.DS_LOAD_B32)
+    self.assertIsNotNone(pcode)
+    self.assertIn('RETURN_DATA', pcode)
+    self.assertIn('MEM[', pcode)
+
+  def test_ds_store_b32_pcode(self):
+    """Test DS_STORE_B32 pcode is parseable."""
+    pcode = PCODE.get(DSOp.DS_STORE_B32)
+    self.assertIsNotNone(pcode)
+    self.assertIn('MEM[', pcode)
+    self.assertIn('DATA', pcode)
+
+  def test_mem_read_parsing(self):
+    """Test MEM[addr].type read expression parsing."""
+    # Create a mock LDS buffer
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
+    addr = UOp.const(dtypes.uint32, 0)
+    vars = {'_lds': lds, 'ADDR': addr, 'OFFSET': UOp.const(dtypes.uint32, 0)}
+
+    result = parse_expr('MEM[ADDR + OFFSET].b32', vars)
+    # Should be an INDEX operation into LDS
+    self.assertIsNotNone(result)
+
+  def test_ds_store_2addr_b32_parsing(self):
+    """Test DS_STORE_2ADDR_B32 pcode parsing produces MEM writes."""
+    pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
+    self.assertIsNotNone(pcode)
+    srcs = {
+      'ADDR': UOp.const(dtypes.uint32, 0),
+      'OFFSET0': UOp.const(dtypes.uint32, 0),
+      'OFFSET1': UOp.const(dtypes.uint32, 1),
+      'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
+      'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
+    }
+    srcs['laneId'] = UOp.const(dtypes.uint32, 0)
+    _, assigns = parse_pcode(pcode, srcs)
+    # Should have 2 MEM write assignments
+    self.assertEqual(len(assigns), 2)
+    for dest, val in assigns:
+      self.assertTrue(dest.startswith('MEM['))
+      # val should be (addr, write_val) tuple
+      self.assertIsInstance(val, tuple)
+      self.assertEqual(len(val), 2)
+
+  def test_ds_load_2addr_b32_parsing(self):
+    """Test DS_LOAD_2ADDR_B32 pcode parsing produces RETURN_DATA assignments."""
+    pcode = PCODE.get(DSOp.DS_LOAD_2ADDR_B32)
+    self.assertIsNotNone(pcode)
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
+    srcs = {
+      'ADDR': UOp.const(dtypes.uint32, 0),
+      'OFFSET0': UOp.const(dtypes.uint32, 0),
+      'OFFSET1': UOp.const(dtypes.uint32, 1),
+      '_lds': lds,
+    }
+    srcs['laneId'] = UOp.const(dtypes.uint32, 0)
+    _, assigns = parse_pcode(pcode, srcs)
+    # Should have 2 RETURN_DATA assignments
+    self.assertEqual(len(assigns), 2)
+    self.assertEqual(assigns[0][0], 'RETURN_DATA[31:0]')
+    self.assertEqual(assigns[1][0], 'RETURN_DATA[63:32]')
+
+  def test_ds_store_address_calculation(self):
+    """Test DS_STORE_2ADDR_B32 calculates correct addresses (offset * 4)."""
+    pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
+    srcs = {
+      'ADDR': UOp.const(dtypes.uint32, 100),
+      'OFFSET0': UOp.const(dtypes.uint32, 2),
+      'OFFSET1': UOp.const(dtypes.uint32, 5),
+      'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
+      'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
+    }
+    srcs['laneId'] = UOp.const(dtypes.uint32, 0)
+    _, assigns = parse_pcode(pcode, srcs)
+    # Check addresses: 100 + 2*4 = 108, 100 + 5*4 = 120
+    addr0, _ = assigns[0][1]
+    addr1, _ = assigns[1][1]
+    self.assertEqual(addr0.simplify().arg, 108)
+    self.assertEqual(addr1.simplify().arg, 120)
+
+  def test_ds_store_data_values(self):
+    """Test DS_STORE_2ADDR_B32 uses correct data values."""
+    pcode = PCODE.get(DSOp.DS_STORE_2ADDR_B32)
+    srcs = {
+      'ADDR': UOp.const(dtypes.uint32, 0),
+      'OFFSET0': UOp.const(dtypes.uint32, 0),
+      'OFFSET1': UOp.const(dtypes.uint32, 1),
+      'DATA': UOp.const(dtypes.uint32, 0xAAAAAAAA),
+      'DATA2': UOp.const(dtypes.uint32, 0xBBBBBBBB),
+    }
+    srcs['laneId'] = UOp.const(dtypes.uint32, 0)
+    _, assigns = parse_pcode(pcode, srcs)
+    _, val0 = assigns[0][1]
+    _, val1 = assigns[1][1]
+    # DATA[31:0] should preserve the value
+    self.assertEqual(val0.simplify().arg, 0xAAAAAAAA)
+    self.assertEqual(val1.simplify().arg, 0xBBBBBBBB)
+
+class TestConditionalParsing(unittest.TestCase):
+  """Test conditional (if/elsif/else) pcode parsing."""
+
+  def test_ternary_in_assignment(self):
+    """Test parsing ternary expression (which becomes WHERE)."""
+    # S_CSELECT_B32: D0.u32 = SCC ? S0.u32 : S1.u32
+    pcode = PCODE[SOP2Op.S_CSELECT_B32]
+    s0 = UOp.const(dtypes.uint32, 10)
+    s1 = UOp.const(dtypes.uint32, 20)
+    scc = UOp.const(dtypes.uint32, 1)
+    vars, assigns = parse_pcode(pcode, {'S0': s0, 'S1': s1, 'SCC': scc})
+    self.assertEqual(len(assigns), 1)
+    dest, val = assigns[0]
+    self.assertTrue(dest.startswith('D0'))
+    # Result should be a WHERE (ternary becomes WHERE)
+    self.assertEqual(val.op, Ops.WHERE)
+
+class TestAllPcode(unittest.TestCase):
+  """Test that all pcode from all architectures can be parsed."""
+
+  def _make_srcs(self):
+    """Create dummy source variables for pcode parsing."""
+    u32, u64 = lambda v=0: UOp.const(dtypes.uint32, v), lambda v=0: UOp.const(dtypes.uint64, v)
+    lds = UOp(Ops.DEFINE_GLOBAL, dtypes.uint32.ptr(16384), arg=3)
+    return {'laneId': u32(), 'laneID': u32(), 'S0': u32(), 'S1': u32(), 'S2': u32(), 'S3': u32(), 'SRC0': u32(),
+            'D0': u32(), 'D1': u32(), 'DST': u32(), 'VDST': u32(), 'SDST': u32(),
+            'VCC': u64(), 'VCCZ': u32(), 'EXEC': u64(), 'EXEC_LO': u32(), 'EXECZ': u32(), 'SCC': u32(),
+            'SIMM16': u32(), 'SIMM32': u32(), 'OFFSET': u32(), 'OFFSET0': u32(), 'OFFSET1': u32(), 'offset1': u32(),
+            'ADDR': u32(), 'ADDR_BASE': u32(), 'TADDR': u32(), 'DATA': u32(), 'DATA0': u32(), 'DATA1': u32(), 'DATA2': u32(),
+            'VDATA': u32(), 'VDATA0': u32(), 'VDATA1': u32(), 'VDATA2': u32(), 'VDATA3': u32(),
+            'OPSEL': u32(), 'OPSEL_HI': u32(), 'NEG': u32(), 'NEG_HI': u32(), 'CLAMP': u32(),
+            'M0': u32(), 'PC': u64(), 'DENORM': u32(1), 'ROUND_MODE': u32(), 'ROUND_TOWARD_ZERO': u32(), 'ROUND_NEAREST_EVEN': u32(), 'WAVE_STATUS': u32(),
+            'MAX_FLOAT_F32': u32(0x7f7fffff), 'Unsigned': u32(1), 'clampedLOD': u32(),
+            '_lds': lds, '_vmem': lds, '_active': UOp.const(dtypes.bool, True)}
+
+  def _parse_all_pcode(self, pcode_dict, arch: str, min_pct: float):
+    """Parse all pcode. RuntimeError = parser limitation (ok), other exceptions = real bugs."""
+    srcs = self._make_srcs()
+    passed, skipped, errors = 0, 0, defaultdict(list)
+    for op, pcode in pcode_dict.items():
+      try:
+        parse_pcode(pcode, srcs)
+        passed += 1
+      except RuntimeError as e: skipped += 1; errors[str(e)].append(op.name)
+      except Exception as e: self.fail(f"[{arch}] {op.name}: {e}\nPcode: {pcode[:200]}")
+    total = len(pcode_dict)
+    pct = 100 * passed / total
+    print(f"{arch}: {passed}/{total} ({pct:.1f}%) parsed, {skipped} skipped")
+    if DEBUG >= 2:
+      for err, ops in sorted(errors.items(), key=lambda x: -len(x[1])):
+        print(f"  {err}: {', '.join(ops[:5])}{'...' if len(ops) > 5 else ''} ({len(ops)})")
+    self.assertGreaterEqual(pct, min_pct, f"[{arch}] {pct:.1f}% < {min_pct}% threshold")
+
+  def test_parse_all_cdna_pcode(self):
+    from extra.assembly.amd.autogen.cdna.str_pcode import PCODE as CDNA_PCODE
+    self._parse_all_pcode(CDNA_PCODE, "CDNA", min_pct=60)
+
+  def test_parse_all_rdna3_pcode(self):
+    from extra.assembly.amd.autogen.rdna3.str_pcode import PCODE as RDNA3_PCODE
+    self._parse_all_pcode(RDNA3_PCODE, "RDNA3", min_pct=90)
+
+  def test_parse_all_rdna4_pcode(self):
+    from extra.assembly.amd.autogen.rdna4.str_pcode import PCODE as RDNA4_PCODE
+    self._parse_all_pcode(RDNA4_PCODE, "RDNA4", min_pct=65)
+
+if __name__ == "__main__":
+  unittest.main()

extra/assembly/amd/test/test_formats.py

@@ -7,7 +7,7 @@ import unittest
 from extra.assembly.amd.autogen.rdna3.ins import *
 from extra.assembly.amd.dsl import VCC_HI, EXEC_LO, NULL
 OFF = NULL  # OFF is alias for NULL
-from extra.assembly.amd.decode import detect_format
+from extra.assembly.amd import detect_format
 
 
 class TestDS(unittest.TestCase):
@@ -213,8 +213,8 @@ class TestDetectFormat(unittest.TestCase):
     self.assertEqual(detect_format(ds_store_b32(v[0], v[1]).to_bytes()), DS)
 
   def test_detect_flat(self):
-    self.assertEqual(detect_format(global_load_b32(vdst=v[0], addr=v[1:2], saddr=NULL).to_bytes()), FLAT)
-    self.assertEqual(detect_format(global_store_b32(addr=v[0:1], data=v[2], saddr=NULL).to_bytes()), FLAT)
+    self.assertEqual(detect_format(global_load_b32(vdst=v[0], addr=v[1:2], saddr=NULL).to_bytes()), GLOBAL)
+    self.assertEqual(detect_format(global_store_b32(addr=v[0:1], data=v[2], saddr=NULL).to_bytes()), GLOBAL)
 
   def test_detect_vopd(self):
     inst = VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[0], vdsty=v[1], srcx0=0, srcy0=0)

extra/assembly/amd/test/test_handwritten.py

@@ -6,17 +6,21 @@ from extra.assembly.amd.autogen.rdna3.ins import *
 from extra.assembly.amd.dsl import Inst
 from extra.assembly.amd.test.test_roundtrip import compile_asm
 
-class TestIntegration(unittest.TestCase):
+class IntegrationTestBase(unittest.TestCase):
   inst: Inst
+  arch: str
   def tearDown(self):
     if not hasattr(self, 'inst'): return
     b = self.inst.to_bytes()
     st = self.inst.disasm()
     # Test that the instruction can be compiled by LLVM and produces the same bytes
     desc = f"{st:25s} {self.inst} {b!r}"
-    self.assertEqual(b, compile_asm(st), desc)
+    self.assertEqual(b, compile_asm(st, arch=self.arch), desc)
     print(desc)
 
+class TestIntegration(IntegrationTestBase):
+  arch: str = "rdna3"
+
   def test_wmma(self):
     self.inst = v_wmma_f32_16x16x16_f16(v[0:7], v[184:191], v[136:143], v[0:7])
 
@@ -124,6 +128,17 @@ class TestIntegration(unittest.TestCase):
     int_inst = s_mov_b32(s[0], struct.unpack("I", struct.pack("f", 1337.0))[0])
     self.assertEqual(self.inst, int_inst)
 
+class TestIntegrationCDNA(IntegrationTestBase):
+  arch = "cdna"
+
+  def test_mfma(self):
+    from extra.assembly.amd.autogen.cdna.ins import v_mfma_f32_16x16x16_f16
+    self.inst = v_mfma_f32_16x16x16_f16(v[0:3], v[0:1], v[0:1], 0)
+
+  def test_mfma_fp8(self):
+    from extra.assembly.amd.autogen.cdna.ins import v_mfma_f32_16x16x128_f8f6f4
+    self.inst = v_mfma_f32_16x16x128_f8f6f4(v[0:3], v[0:5], v[0:5], 1, cbsz=2, blgp=2)
+
 class TestRegisterSliceSyntax(unittest.TestCase):
   """
   Issue: Register slice syntax should use AMD assembly convention (inclusive end).

extra/assembly/amd/test/test_llvm.py

@@ -11,8 +11,8 @@ Only compute-relevant instruction formats are tested. Graphics-only formats not
 import unittest, re, subprocess, functools
 from tinygrad.helpers import fetch
 from extra.assembly.amd.disasm import disasm
-from extra.assembly.amd.decode import decode_inst, detect_format
-from extra.assembly.amd.test.helpers import get_llvm_mc
+from extra.assembly.amd import decode_inst, detect_format
+from extra.assembly.amd.test.helpers import get_llvm_mc, get_target, get_mattr
 
 LLVM_BASE = "https://raw.githubusercontent.com/llvm/llvm-project/llvmorg-21.1.0/llvm/test/MC/AMDGPU"
 
@@ -25,13 +25,15 @@ RDNA_FILES = ['gfx11_asm_sop1.s', 'gfx11_asm_sop2.s', 'gfx11_asm_sopp.s', 'gfx11
   'gfx11_asm_wmma.s', 'gfx11_asm_vop3_features.s', 'gfx11_asm_vop3p_features.s', 'gfx11_asm_vopd_features.s',
   'gfx11_asm_vop3_alias.s', 'gfx11_asm_vop3p_alias.s', 'gfx11_asm_vopc_alias.s', 'gfx11_asm_vopcx_alias.s', 'gfx11_asm_vinterp_alias.s',
   'gfx11_asm_smem_alias.s']
-# CDNA (gfx9/gfx90a/gfx942) test files for compute instructions
+# CDNA (gfx9/gfx90a/gfx942/gfx950) test files for compute instructions
 # Excluded: gfx9_asm_mubuf.s, gfx9_asm_mtbuf.s, gfx90a_ldst_acc.s (has MIMG mixed in)
+# Exclude gfx90a: 'gfx90a_asm_features.s', 'mai-gfx90a.s',
+# Exclude gfx950: 'gfx950_asm_features.s' (disasm error)
 CDNA_FILES = ['gfx9_asm_sop1.s', 'gfx9_asm_sop2.s', 'gfx9_asm_sopp.s', 'gfx9_asm_sopk.s', 'gfx9_asm_sopc.s',
   'gfx9_asm_vop1.s', 'gfx9_asm_vop2.s', 'gfx9_asm_vopc.s', 'gfx9_asm_vop3.s', 'gfx9_asm_vop3p.s',
   'gfx9_asm_ds.s', 'gfx9_asm_flat.s', 'gfx9_asm_smem.s',
-  'gfx90a_asm_features.s', 'flat-scratch-gfx942.s', 'gfx942_asm_features.s',
-  'mai-gfx90a.s', 'mai-gfx942.s']
+  'flat-scratch-gfx942.s', 'gfx942_asm_features.s', 'mai-gfx942.s',
+  'gfx950_asm_vop1.s', 'gfx950_asm_read_tr.s', 'mai-gfx950.s']
 # RDNA4 (gfx12) test files for compute instructions
 # Excluded: gfx12_asm_vbuffer_mubuf.s, gfx12_asm_vbuffer_mtbuf.s, gfx12_asm_exp.s (graphics-only)
 RDNA4_FILES = ['gfx12_asm_sop1.s', 'gfx12_asm_sop2.s', 'gfx12_asm_sopp.s', 'gfx12_asm_sopk.s', 'gfx12_asm_sopc.s',
@@ -57,8 +59,7 @@ def _parse_llvm_tests(text: str, pattern: str) -> list[tuple[str, bytes]]:
       except ValueError: pass
   return tests
 
-@functools.cache
-def _get_tests(f: str, arch: str) -> list[tuple[str, bytes]]:
+def _get_tests_uncached(f: str, arch: str) -> list[tuple[str, bytes]]:
   text = fetch(f"{LLVM_BASE}/{f}").read_bytes().decode('utf-8', errors='ignore')
   if arch == "rdna3":
     # Match GFX11 and W32 only (wavefront32 mode)
@@ -66,25 +67,54 @@ def _get_tests(f: str, arch: str) -> list[tuple[str, bytes]]:
   elif arch == "rdna4":
     # Match GFX12 (but not GFX1250) and W32 only (wavefront32 mode)
     tests = _parse_llvm_tests(text, r'(?:GFX12(?!50)|W32)')
-  elif 'gfx90a' in f or 'gfx942' in f:
-    tests = _parse_llvm_tests(text, r'(?:GFX90A|GFX942)')
+  elif 'gfx90a' in f or 'gfx942' in f or 'gfx950' in f:
+    tests = _parse_llvm_tests(text, r'(?:GFX90A|GFX942|GFX950)')
   else:
     tests = _parse_llvm_tests(text, r'(?:VI9|GFX9|CHECK)')
+  # Exclude v_interp_* (graphics-only, not on CDNA)
+  if arch == "cdna": tests = [(asm, data) for asm, data in tests if not asm.startswith('v_interp_')]
+  # Filter out tests where original ASM isn't valid on target (e.g., gfx9 tests with gfx942/gfx950 constraints)
+  if arch == "cdna" and not ('gfx942' in f or 'gfx950' in f or 'gfx90a' in f): tests = _filter_valid_asm(tests, arch)
   return tests
 
-def _compile_asm_batch(instrs: list[str], arch: str = "rdna3") -> list[bytes]:
+@functools.cache
+def _get_tests(f: str, arch: str) -> list[tuple[str, bytes]]: return _get_tests_uncached(f, arch)
+
+def _compile_asm_batch(instrs: list[str], arch: str = "rdna3", mcpu: str|None = None) -> list[bytes]:
   if not instrs: return []
-  mcpu = {'rdna3': 'gfx1100', 'rdna4': 'gfx1200'}.get(arch, 'gfx1100')
-  result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={mcpu}', '-mattr=+real-true16,+wavefrontsize32', '-show-encoding'],
+  mcpu, mattr = mcpu or get_target(arch), get_mattr(arch)
+  result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={mcpu}', f'-mattr={mattr}', '-show-encoding'],
     input=".text\n" + "\n".join(instrs) + "\n", capture_output=True, text=True, timeout=30)
   if result.returncode != 0: raise RuntimeError(f"llvm-mc failed: {result.stderr.strip()}")
   return [bytes.fromhex(line.split('encoding:')[1].strip()[1:-1].replace('0x', '').replace(',', '').replace(' ', ''))
           for line in result.stdout.split('\n') if 'encoding:' in line]
 
+def _filter_valid_asm(tests: list[tuple[str, bytes]], arch: str) -> list[tuple[str, bytes]]:
+  """Filter out tests where the original ASM isn't valid on the target (e.g., gfx9 tests with gfx942/gfx950 constraints)."""
+  if not tests: return []
+  mcpu = get_target(arch)
+  # Batch assemble all instructions, parse stderr to find which lines failed
+  instrs = [asm for asm, _ in tests]
+  result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={mcpu}', '-show-encoding'],
+    input=".text\n" + "\n".join(instrs) + "\n", capture_output=True, text=True, timeout=30)
+  # Parse error lines from stderr (format: "<stdin>:N:..." where N is 1-indexed, line 1 is ".text")
+  failed_lines = set()
+  for line in result.stderr.split('\n'):
+    if m := re.match(r'<stdin>:(\d+):', line): failed_lines.add(int(m.group(1)) - 1)  # -1 for .text, so line 2 -> index 1 -> tests[0]
+  # Also filter out tests where LLVM roundtrip doesn't match original (reserved bits set in original)
+  valid = [(asm, data) for i, (asm, data) in enumerate(tests) if (i + 1) not in failed_lines]
+  if not valid: return []
+  llvm_result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={mcpu}', '-show-encoding'],
+    input=".text\n" + "\n".join(asm for asm, _ in valid) + "\n", capture_output=True, text=True, timeout=30)
+  llvm_bytes = [bytes.fromhex(line.split('encoding:')[1].strip()[1:-1].replace('0x', '').replace(',', '').replace(' ', ''))
+                for line in llvm_result.stdout.split('\n') if 'encoding:' in line]
+  return [(asm, data) for (asm, data), lb in zip(valid, llvm_bytes) if lb == data]
+
 def _make_test(f: str, arch: str, test_type: str):
   def test(self):
     tests = _get_tests(f, arch)
     name = f"{arch}_{test_type}_{f}"
+    mcpu = "gfx942" if arch == "cdna" and "gfx942" in f else get_target(arch)
     if test_type == "roundtrip":
       passed, skipped = 0, 0
       for _, data in tests:
@@ -94,21 +124,44 @@ def _make_test(f: str, arch: str, test_type: str):
           passed += 1
         except ValueError: skipped += 1  # skip invalid opcodes not in enum
       print(f"{name}: {passed} passed, {skipped} skipped")
-      if arch in ("rdna3", "rdna4"):
-        self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
+      self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
+    elif test_type == "repr":
+      # Test that eval(repr(inst)) reproduces the instruction
+      if arch == "rdna3": import extra.assembly.amd.autogen.rdna3.ins as ins
+      elif arch == "rdna4": import extra.assembly.amd.autogen.rdna4.ins as ins
+      elif arch == "cdna": import extra.assembly.amd.autogen.cdna.ins as ins
+      ns = {k: getattr(ins, k) for k in dir(ins) if not k.startswith('_')}
+      passed, skipped = 0, 0
+      for _, data in tests:
+        try:
+          decoded = detect_format(data, arch).from_bytes(data)
+          if decoded.to_bytes()[:len(data)] != data: skipped += 1; continue  # skip if binary roundtrip fails
+          r = repr(decoded)
+          try:
+            decoded2 = eval(r, ns)  # noqa: S307
+            if decoded == decoded2: passed += 1
+            else: skipped += 1
+          except Exception: skipped += 1
+        except ValueError: skipped += 1
+      print(f"{name}: {passed} passed, {skipped} skipped")
+      self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
     elif test_type == "disasm":
       to_test = []
       for _, data in tests:
         try:
           decoded = decode_inst(data, arch)
+          enc = decoded.to_bytes()[:len(data)]
           # Skip if roundtrip fails, disasm fails, or op_name is missing (disasm starts with space)
-          if decoded.to_bytes()[:len(data)] == data and (d := disasm(decoded)) and not d.startswith(' '): to_test.append((data, d))
+          if enc == data and (d := disasm(decoded)) and not d.startswith(' '): to_test.append((enc, d))
         except: pass
       skipped = len(tests) - len(to_test)
       print(f"{name}: {len(to_test)} passed, {skipped} skipped")
-      if arch in ("rdna3", "rdna4"):
-        self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
-        for (data, _), llvm in zip(to_test, _compile_asm_batch([t[1] for t in to_test], arch)): self.assertEqual(llvm, data)
+      self.assertEqual(skipped, 0, f"{name}: {skipped} tests skipped, expected 0")
+      # Compare disasm->reassemble with original encoding (filter reserved bit cases where LLVM can't reproduce)
+      llvm_bytes = _compile_asm_batch([t[1] for t in to_test], arch, mcpu)
+      valid = [(enc, d, llvm) for (enc, d), llvm in zip(to_test, llvm_bytes) if llvm == enc]
+      print(f"{name}: {len(valid)}/{len(to_test)} matched LLVM encoding")
+      for enc, _, llvm in valid: self.assertEqual(llvm, enc)
   return test
 
 class TestLLVM(unittest.TestCase): pass
@@ -116,12 +169,15 @@ class TestLLVM(unittest.TestCase): pass
 for f in RDNA_FILES:
   setattr(TestLLVM, f"test_rdna3_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "roundtrip"))
   setattr(TestLLVM, f"test_rdna3_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "disasm"))
+  setattr(TestLLVM, f"test_rdna3_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna3", "repr"))
 for f in CDNA_FILES:
   setattr(TestLLVM, f"test_cdna_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "roundtrip"))
   setattr(TestLLVM, f"test_cdna_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "disasm"))
+  setattr(TestLLVM, f"test_cdna_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "cdna", "repr"))
 for f in RDNA4_FILES:
   setattr(TestLLVM, f"test_rdna4_roundtrip_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "roundtrip"))
   setattr(TestLLVM, f"test_rdna4_disasm_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "disasm"))
+  setattr(TestLLVM, f"test_rdna4_repr_{f.replace('.s', '').replace('-', '_')}", _make_test(f, "rdna4", "repr"))
 
 if __name__ == "__main__":
   unittest.main()

extra/assembly/amd/test/test_pcode.py

@@ -1,403 +0,0 @@
-#!/usr/bin/env python3
-"""Tests for the RDNA3 pseudocode DSL."""
-import unittest
-from extra.assembly.amd.pcode import (Reg, TypedView, TypedView, MASK32, MASK64,
-                                       _f32, _i32, _f16, _i16, f32_to_f16, isNAN, _bf16, _ibf16, bf16_to_f32, f32_to_bf16,
-                                       BYTE_PERMUTE, v_sad_u8, v_msad_u8, _compile_pseudocode, _expr, compile_pseudocode)
-from extra.assembly.amd.test.helpers import ExecContext
-from extra.assembly.amd.autogen.rdna3.str_pcode import PCODE
-from extra.assembly.amd.autogen.rdna3.enum import VOP3SDOp, VOPCOp
-
-# Compile pseudocode functions on demand for regression tests
-_VOP3SDOp_V_DIV_SCALE_F32 = compile_pseudocode('VOP3SDOp', 'V_DIV_SCALE_F32', PCODE[VOP3SDOp.V_DIV_SCALE_F32])
-_VOPCOp_V_CMP_CLASS_F32 = compile_pseudocode('VOPCOp', 'V_CMP_CLASS_F32', PCODE[VOPCOp.V_CMP_CLASS_F32_E32])
-
-class TestReg(unittest.TestCase):
-  def test_u32_read(self):
-    r = Reg(0xDEADBEEF)
-    self.assertEqual(int(r.u32), 0xDEADBEEF)
-
-  def test_u32_write(self):
-    r = Reg(0)
-    r.u32 = 0x12345678
-    self.assertEqual(r._val, 0x12345678)
-
-  def test_f32_read(self):
-    r = Reg(0x40400000)  # 3.0f
-    self.assertAlmostEqual(float(r.f32), 3.0)
-
-  def test_f32_write(self):
-    r = Reg(0)
-    r.f32 = 3.0
-    self.assertEqual(r._val, 0x40400000)
-
-  def test_i32_signed(self):
-    r = Reg(0xFFFFFFFF)  # -1 as signed
-    self.assertEqual(int(r.i32), -1)
-
-  def test_u64(self):
-    r = Reg(0xDEADBEEFCAFEBABE)
-    self.assertEqual(int(r.u64), 0xDEADBEEFCAFEBABE)
-
-  def test_f64(self):
-    r = Reg(0x4008000000000000)  # 3.0 as f64
-    self.assertAlmostEqual(float(r.f64), 3.0)
-
-class TestTypedView(unittest.TestCase):
-  def test_bit_slice(self):
-    r = Reg(0xDEADBEEF)
-    # Slices return TypedView which supports .u32, .u16 etc (matching pseudocode like S1.u32[1:0].u32)
-    self.assertEqual(r.u32[7:0].u32, 0xEF)
-    self.assertEqual(r.u32[15:8].u32, 0xBE)
-    self.assertEqual(r.u32[23:16].u32, 0xAD)
-    self.assertEqual(r.u32[31:24].u32, 0xDE)
-    # Also works with int() for arithmetic
-    self.assertEqual(int(r.u32[7:0]), 0xEF)
-
-  def test_single_bit_read(self):
-    r = Reg(0b11010101)
-    self.assertEqual(r.u32[0], 1)
-    self.assertEqual(r.u32[1], 0)
-    self.assertEqual(r.u32[2], 1)
-    self.assertEqual(r.u32[3], 0)
-
-  def test_single_bit_write(self):
-    r = Reg(0)
-    r.u32[5] = 1
-    r.u32[3] = 1
-    self.assertEqual(r._val, 0b00101000)
-
-  def test_nested_bit_access(self):
-    # S0.u32[S1.u32[4:0]] - access bit at position from another register
-    s0 = Reg(0b11010101)
-    s1 = Reg(3)
-    bit_pos = s1.u32[4:0]  # TypedView, int value = 3
-    bit_val = s0.u32[int(bit_pos)]  # bit 3 of s0 = 0
-    self.assertEqual(int(bit_pos), 3)
-    self.assertEqual(bit_val, 0)
-
-  def test_arithmetic(self):
-    r1 = Reg(0x40400000)  # 3.0f
-    r2 = Reg(0x40800000)  # 4.0f
-    result = r1.f32 + r2.f32
-    self.assertAlmostEqual(result, 7.0)
-
-  def test_comparison(self):
-    r1 = Reg(5)
-    r2 = Reg(3)
-    self.assertTrue(r1.u32 > r2.u32)
-    self.assertFalse(r1.u32 < r2.u32)
-    self.assertTrue(r1.u32 != r2.u32)
-
-class TestTypedView(unittest.TestCase):
-  def test_slice_read(self):
-    r = Reg(0x56781234)
-    self.assertEqual(r[15:0].u16, 0x1234)
-    self.assertEqual(r[31:16].u16, 0x5678)
-
-  def test_slice_write(self):
-    r = Reg(0)
-    r[15:0].u16 = 0x1234
-    r[31:16].u16 = 0x5678
-    self.assertEqual(r._val, 0x56781234)
-
-  def test_slice_f16(self):
-    r = Reg(0)
-    r[15:0].f16 = 3.0
-    self.assertAlmostEqual(_f16(r._val & 0xffff), 3.0, places=2)
-
-class TestCompiler(unittest.TestCase):
-  def test_ternary(self):
-    result = _expr("a > b ? 1 : 0")
-    self.assertIn("if", result)
-    self.assertIn("else", result)
-
-  def test_type_prefix_strip(self):
-    self.assertEqual(_expr("1'0U"), "0")
-    self.assertEqual(_expr("32'1"), "1")
-    self.assertEqual(_expr("16'0xFFFF"), "0xFFFF")
-
-  def test_suffix_strip(self):
-    self.assertEqual(_expr("0ULL"), "0")
-    self.assertEqual(_expr("1LL"), "1")
-    self.assertEqual(_expr("5U"), "5")
-    self.assertEqual(_expr("3.14F"), "3.14")
-
-  def test_boolean_ops(self):
-    self.assertIn("and", _expr("a && b"))
-    self.assertIn("or", _expr("a || b"))
-    self.assertIn("!=", _expr("a <> b"))
-
-  def test_pack16(self):
-    result = _expr("{ a, b }")
-    self.assertIn("_pack", result)
-
-  def test_type_cast_strip(self):
-    self.assertEqual(_expr("64'U(x)"), "(x)")
-    self.assertEqual(_expr("32'I(y)"), "(y)")
-
-class TestExecContext(unittest.TestCase):
-  def test_float_add(self):
-    ctx = ExecContext(s0=0x40400000, s1=0x40800000)  # 3.0f, 4.0f
-    ctx.D0.f32 = ctx.S0.f32 + ctx.S1.f32
-    self.assertAlmostEqual(_f32(ctx.D0._val), 7.0)
-
-  def test_float_mul(self):
-    ctx = ExecContext(s0=0x40400000, s1=0x40800000)  # 3.0f, 4.0f
-    ctx.run("D0.f32 = S0.f32 * S1.f32")
-    self.assertAlmostEqual(_f32(ctx.D0._val), 12.0)
-
-  def test_scc_comparison(self):
-    ctx = ExecContext(s0=42, s1=42)
-    ctx.run("SCC = S0.u32 == S1.u32")
-    self.assertEqual(ctx.SCC._val, 1)
-
-  def test_scc_comparison_false(self):
-    ctx = ExecContext(s0=42, s1=43)
-    ctx.run("SCC = S0.u32 == S1.u32")
-    self.assertEqual(ctx.SCC._val, 0)
-
-  def test_ternary(self):
-    code = _compile_pseudocode("D0.u32 = S0.u32 > S1.u32 ? 1'1U : 1'0U")
-    ctx = ExecContext(s0=5, s1=3)
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val, 1)
-
-  def test_pack(self):
-    code = _compile_pseudocode("D0 = { S1[15:0].u16, S0[15:0].u16 }")
-    ctx = ExecContext(s0=0x1234, s1=0x5678)
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val, 0x56781234)
-
-  def test_tmp_with_typed_access(self):
-    code = _compile_pseudocode("""tmp = S0.u32 + S1.u32
-D0.u32 = tmp.u32""")
-    ctx = ExecContext(s0=100, s1=200)
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val, 300)
-
-  def test_s_add_u32_pattern(self):
-    # Real pseudocode pattern from S_ADD_U32
-    code = _compile_pseudocode("""tmp = 64'U(S0.u32) + 64'U(S1.u32)
-SCC = tmp >= 0x100000000ULL ? 1'1U : 1'0U
-D0.u32 = tmp.u32""")
-    # Test overflow case
-    ctx = ExecContext(s0=0xFFFFFFFF, s1=0x00000001)
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val, 0)  # Wraps to 0
-    self.assertEqual(ctx.SCC._val, 1)  # Carry set
-
-  def test_s_add_u32_no_overflow(self):
-    code = _compile_pseudocode("""tmp = 64'U(S0.u32) + 64'U(S1.u32)
-SCC = tmp >= 0x100000000ULL ? 1'1U : 1'0U
-D0.u32 = tmp.u32""")
-    ctx = ExecContext(s0=100, s1=200)
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val, 300)
-    self.assertEqual(ctx.SCC._val, 0)  # No carry
-
-  def test_vcc_lane_read(self):
-    ctx = ExecContext(vcc=0b1010, lane=1)
-    # Lane 1 is set
-    self.assertEqual(ctx.VCC.u64[1], 1)
-    self.assertEqual(ctx.VCC.u64[2], 0)
-
-  def test_vcc_lane_write(self):
-    ctx = ExecContext(vcc=0, lane=0)
-    ctx.VCC.u64[3] = 1
-    ctx.VCC.u64[1] = 1
-    self.assertEqual(ctx.VCC._val, 0b1010)
-
-  def test_for_loop(self):
-    # CTZ pattern - find first set bit
-    code = _compile_pseudocode("""tmp = -1
-for i in 0 : 31 do
-  if S0.u32[i] == 1 then
-    tmp = i
-  endif
-endfor
-D0.i32 = tmp""")
-    ctx = ExecContext(s0=0b1000)  # Bit 3 is set
-    ctx.run(code)
-    self.assertEqual(ctx.D0._val & MASK32, 3)
-
-  def test_result_dict(self):
-    ctx = ExecContext(s0=5, s1=3)
-    ctx.D0.u32 = 42
-    ctx.SCC._val = 1
-    result = ctx.result()
-    self.assertEqual(result['d0'], 42)
-    self.assertEqual(result['scc'], 1)
-
-class TestPseudocodeRegressions(unittest.TestCase):
-  """Regression tests for pseudocode instruction emulation bugs."""
-
-  def test_v_div_scale_f32_vcc_always_returned(self):
-    """V_DIV_SCALE_F32 must always return VCC, even when VCC=0 (no scaling needed).
-    Bug: when VCC._val == vcc (both 0), VCC wasn't returned, so VCC bits weren't written.
-    This caused division to produce wrong results for multiple lanes."""
-    # Normal case: 1.0 / 3.0, no scaling needed, VCC should be 0
-    s0 = 0x3f800000  # 1.0
-    s1 = 0x40400000  # 3.0
-    s2 = 0x3f800000  # 1.0 (numerator)
-    result = _VOP3SDOp_V_DIV_SCALE_F32(s0, s1, s2, 0, 0, 0, 0, 0xffffffff, 0, None)
-    # Must always have VCC in result
-    self.assertIn('VCC', result, "V_DIV_SCALE_F32 must always return VCC")
-    self.assertEqual(result['VCC'] & 1, 0, "VCC lane 0 should be 0 when no scaling needed")
-
-  def test_v_cmp_class_f32_detects_quiet_nan(self):
-    """V_CMP_CLASS_F32 must correctly identify quiet NaN vs signaling NaN.
-    Bug: isQuietNAN and isSignalNAN both used math.isnan which can't distinguish them."""
-    quiet_nan = 0x7fc00000   # quiet NaN: exponent=255, bit22=1
-    signal_nan = 0x7f800001  # signaling NaN: exponent=255, bit22=0
-    # Test quiet NaN detection (bit 1 in mask)
-    s1_quiet = 0b0000000010  # bit 1 = quiet NaN
-    result = _VOPCOp_V_CMP_CLASS_F32(quiet_nan, s1_quiet, 0, 0, 0, 0, 0, 0xffffffff, 0, None)
-    self.assertEqual(result['D0'] & 1, 1, "Should detect quiet NaN with quiet NaN mask")
-    # Test signaling NaN detection (bit 0 in mask)
-    s1_signal = 0b0000000001  # bit 0 = signaling NaN
-    result = _VOPCOp_V_CMP_CLASS_F32(signal_nan, s1_signal, 0, 0, 0, 0, 0, 0xffffffff, 0, None)
-    self.assertEqual(result['D0'] & 1, 1, "Should detect signaling NaN with signaling NaN mask")
-    # Test that quiet NaN doesn't match signaling NaN mask
-    result = _VOPCOp_V_CMP_CLASS_F32(quiet_nan, s1_signal, 0, 0, 0, 0, 0, 0xffffffff, 0, None)
-    self.assertEqual(result['D0'] & 1, 0, "Quiet NaN should not match signaling NaN mask")
-    # Test that signaling NaN doesn't match quiet NaN mask
-    result = _VOPCOp_V_CMP_CLASS_F32(signal_nan, s1_quiet, 0, 0, 0, 0, 0, 0xffffffff, 0, None)
-    self.assertEqual(result['D0'] & 1, 0, "Signaling NaN should not match quiet NaN mask")
-
-  def testisNAN_with_typed_view(self):
-    """isNAN must work with TypedView objects, not just Python floats.
-    Bug: isNAN checked isinstance(x, float) which returned False for TypedView."""
-    nan_reg = Reg(0x7fc00000)  # quiet NaN
-    normal_reg = Reg(0x3f800000)  # 1.0
-    inf_reg = Reg(0x7f800000)  # +inf
-    self.assertTrue(isNAN(nan_reg.f32), "isNAN should return True for NaN TypedView")
-    self.assertFalse(isNAN(normal_reg.f32), "isNAN should return False for normal TypedView")
-    self.assertFalse(isNAN(inf_reg.f32), "isNAN should return False for inf TypedView")
-
-class TestBF16(unittest.TestCase):
-  """Tests for BF16 (bfloat16) support."""
-
-  def test_bf16_conversion(self):
-    """Test bf16 <-> f32 conversion."""
-    # bf16 is just the top 16 bits of f32
-    # 1.0f = 0x3f800000, bf16 = 0x3f80
-    self.assertAlmostEqual(_bf16(0x3f80), 1.0, places=2)
-    self.assertEqual(_ibf16(1.0), 0x3f80)
-    # 2.0f = 0x40000000, bf16 = 0x4000
-    self.assertAlmostEqual(_bf16(0x4000), 2.0, places=2)
-    self.assertEqual(_ibf16(2.0), 0x4000)
-    # -1.0f = 0xbf800000, bf16 = 0xbf80
-    self.assertAlmostEqual(_bf16(0xbf80), -1.0, places=2)
-    self.assertEqual(_ibf16(-1.0), 0xbf80)
-
-  def test_bf16_special_values(self):
-    """Test bf16 special values (inf, nan)."""
-    import math
-    # +inf: f32 = 0x7f800000, bf16 = 0x7f80
-    self.assertTrue(math.isinf(_bf16(0x7f80)))
-    self.assertEqual(_ibf16(float('inf')), 0x7f80)
-    # -inf: f32 = 0xff800000, bf16 = 0xff80
-    self.assertTrue(math.isinf(_bf16(0xff80)))
-    self.assertEqual(_ibf16(float('-inf')), 0xff80)
-    # NaN: quiet NaN bf16 = 0x7fc0
-    self.assertTrue(math.isnan(_bf16(0x7fc0)))
-    self.assertEqual(_ibf16(float('nan')), 0x7fc0)
-
-  def test_bf16_register_property(self):
-    """Test Reg.bf16 property."""
-    r = Reg(0)
-    r.bf16 = 3.0  # 3.0f = 0x40400000, bf16 = 0x4040
-    self.assertEqual(r._val & 0xffff, 0x4040)
-    self.assertAlmostEqual(float(r.bf16), 3.0, places=1)
-
-  def test_bf16_slice_property(self):
-    """Test TypedView.bf16 property."""
-    r = Reg(0x40404040)  # Two bf16 3.0 values
-    self.assertAlmostEqual(r[15:0].bf16, 3.0, places=1)
-    self.assertAlmostEqual(r[31:16].bf16, 3.0, places=1)
-
-class TestBytePermute(unittest.TestCase):
-  """Tests for BYTE_PERMUTE helper function (V_PERM_B32)."""
-
-  def test_byte_select_0_to_7(self):
-    """Test selecting bytes 0-7 from 64-bit data."""
-    # data = {s0, s1} where s0 is bytes 0-3, s1 is bytes 4-7
-    # Combined: 0x0706050403020100 (byte 0 = 0x00, byte 7 = 0x07)
-    data = 0x0706050403020100
-    for i in range(8):
-      self.assertEqual(BYTE_PERMUTE(data, i), i, f"byte {i} should be {i}")
-
-  def test_sign_extend_bytes(self):
-    """Test sign extension selectors 8-11."""
-    # sel 8: sign of byte 1 (bits 15:8)
-    # sel 9: sign of byte 3 (bits 31:24)
-    # sel 10: sign of byte 5 (bits 47:40)
-    # sel 11: sign of byte 7 (bits 63:56)
-    data = 0x8000800080008000  # All relevant bytes have sign bit set
-    self.assertEqual(BYTE_PERMUTE(data, 8), 0xff)
-    self.assertEqual(BYTE_PERMUTE(data, 9), 0xff)
-    self.assertEqual(BYTE_PERMUTE(data, 10), 0xff)
-    self.assertEqual(BYTE_PERMUTE(data, 11), 0xff)
-    data = 0x7f007f007f007f00  # No sign bits set
-    self.assertEqual(BYTE_PERMUTE(data, 8), 0x00)
-    self.assertEqual(BYTE_PERMUTE(data, 9), 0x00)
-    self.assertEqual(BYTE_PERMUTE(data, 10), 0x00)
-    self.assertEqual(BYTE_PERMUTE(data, 11), 0x00)
-
-  def test_constant_zero(self):
-    """Test selector 12 returns 0x00."""
-    self.assertEqual(BYTE_PERMUTE(0xffffffffffffffff, 12), 0x00)
-
-  def test_constant_ff(self):
-    """Test selectors >= 13 return 0xFF."""
-    for sel in [13, 14, 15, 255]:
-      self.assertEqual(BYTE_PERMUTE(0, sel), 0xff, f"sel {sel} should be 0xff")
-
-class TestSADHelpers(unittest.TestCase):
-  """Tests for V_SAD_U8 and V_MSAD_U8 helper functions."""
-
-  def test_v_sad_u8_basic(self):
-    """Test v_sad_u8 with simple values."""
-    # s0 = 0x04030201, s1 = 0x04030201 -> diff = 0 for all bytes
-    result = v_sad_u8(0x04030201, 0x04030201, 0)
-    self.assertEqual(result, 0)
-    # s0 = 0x05040302, s1 = 0x04030201 -> diff = 1+1+1+1 = 4
-    result = v_sad_u8(0x05040302, 0x04030201, 0)
-    self.assertEqual(result, 4)
-
-  def test_v_sad_u8_with_accumulator(self):
-    """Test v_sad_u8 with non-zero accumulator."""
-    # s0 = 0x05040302, s1 = 0x04030201, s2 = 100 -> 4 + 100 = 104
-    result = v_sad_u8(0x05040302, 0x04030201, 100)
-    self.assertEqual(result, 104)
-
-  def test_v_sad_u8_large_diff(self):
-    """Test v_sad_u8 with maximum byte differences."""
-    # s0 = 0xffffffff, s1 = 0x00000000 -> diff = 255*4 = 1020
-    result = v_sad_u8(0xffffffff, 0x00000000, 0)
-    self.assertEqual(result, 1020)
-
-  def test_v_msad_u8_basic(self):
-    """Test v_msad_u8 masks when reference byte is 0."""
-    # s0 = 0x10101010, s1 = 0x00000000 -> all masked, result = 0
-    result = v_msad_u8(0x10101010, 0x00000000, 0)
-    self.assertEqual(result, 0)
-    # s0 = 0x10101010, s1 = 0x01010101 -> diff = |0x10-0x01|*4 = 15*4 = 60
-    result = v_msad_u8(0x10101010, 0x01010101, 0)
-    self.assertEqual(result, 60)
-
-  def test_v_msad_u8_partial_mask(self):
-    """Test v_msad_u8 with partial masking."""
-    # s0 = 0x10101010, s1 = 0x00010001 -> bytes 1 and 3 masked
-    # diff = |0x10-0x01| + |0x10-0x01| = 15 + 15 = 30
-    result = v_msad_u8(0x10101010, 0x00010001, 0)
-    self.assertEqual(result, 30)
-
-  def test_v_msad_u8_with_accumulator(self):
-    """Test v_msad_u8 with non-zero accumulator."""
-    result = v_msad_u8(0x10101010, 0x01010101, 50)
-    self.assertEqual(result, 110)  # 60 + 50
-
-if __name__ == '__main__':
-  unittest.main()

extra/assembly/amd/test/test_pdf.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-"""Test PDF pseudocode extraction from amdxml.py."""
+"""Test PDF pseudocode extraction from generate.py."""
 import unittest
-from extra.assembly.amd.amdxml import extract_pdf_text, extract_pcode, parse_xml, ARCHS, FIXES
+from extra.assembly.amd.generate import extract_pdf_text, extract_pcode, parse_xml, ARCHS, FIXES
 
 EXPECTED_PAGES = {"rdna3": 655, "rdna4": 711, "cdna": 610}
 
@@ -11,7 +11,7 @@ class TestPcodePDF(unittest.TestCase):
     cls.pages = {arch: extract_pdf_text(cfg["pdf"]) for arch, cfg in ARCHS.items()}
     cls.enums = {}
     for arch, cfg in ARCHS.items():
-      _, enums, _, _, _ = parse_xml(cfg["xml"], arch)
+      _, enums, _, _, _, _ = parse_xml(cfg["xml"])
       for fmt, ops in FIXES.get(arch, {}).items(): enums.setdefault(fmt, {}).update(ops)
       cls.enums[arch] = enums
     cls.pcode = {arch: extract_pcode(cls.pages[arch], {n: op for ops in cls.enums[arch].values() for op, n in ops.items()}) for arch in ARCHS}

extra/assembly/amd/test/test_roundtrip.py

@@ -2,15 +2,8 @@
 """Roundtrip tests: generate tinygrad kernels, decode instructions, re-encode, verify match."""
 import unittest, io, sys, re, subprocess, os
 from extra.assembly.amd.dsl import Inst
-from extra.assembly.amd.decode import decode_inst, detect_format
-from extra.assembly.amd.test.helpers import get_llvm_mc, get_llvm_objdump
-
-# arch: (mcpu, mattr)
-ARCH_CONFIG = {
-  'rdna3': ('gfx1100', '+real-true16,+wavefrontsize32'),
-  'rdna4': ('gfx1200', '+real-true16,+wavefrontsize32'),
-  'cdna': ('gfx942', '+wavefrontsize64'),
-}
+from extra.assembly.amd import decode_inst, detect_format
+from extra.assembly.amd.test.helpers import get_llvm_mc, get_llvm_objdump, get_target, get_mattr
 
 def disassemble_lib(lib: bytes, compiler) -> list[tuple[str, bytes]]:
   """Disassemble ELF binary and return list of (instruction_text, machine_code_bytes)."""
@@ -42,8 +35,7 @@ def compile_asm(instr: str, arch: str = 'rdna3') -> bytes:
 def compile_asm_batch(instrs: list[str], arch: str = 'rdna3') -> list[bytes]:
   """Compile multiple instructions with a single llvm-mc call."""
   if not instrs: return []
-  mcpu, mattr = ARCH_CONFIG[arch]
-  result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={mcpu}', f'-mattr={mattr}', '-show-encoding'],
+  result = subprocess.run([get_llvm_mc(), '-triple=amdgcn', f'-mcpu={get_target(arch)}', f'-mattr={get_mattr(arch)}', '-show-encoding'],
                           input=".text\n" + "\n".join(instrs) + "\n", capture_output=True, text=True)
   if result.returncode != 0: raise RuntimeError(f"llvm-mc batch failed: {result.stderr.strip()}")
   encodings = []
@@ -59,7 +51,7 @@ def compile_and_disasm_batch(instrs: list[str], arch: str = 'rdna3') -> list[str
   """Compile instructions with LLVM and get LLVM's disassembly."""
   import tempfile
   if not instrs: return []
-  mcpu, mattr = ARCH_CONFIG[arch]
+  mcpu, mattr = get_target(arch), get_mattr(arch)
   src = ".text\n.globl test\n.p2align 8\n.type test,@function\ntest:\n" + "\n".join(f"  {instr}" for instr in instrs) + "\n"
   with tempfile.NamedTemporaryFile(suffix='.o', delete=False) as f:
     obj_path = f.name
@@ -89,20 +81,23 @@ class TestTinygradKernelRoundtrip(unittest.TestCase):
     3. our disasm() matches LLVM's disassembly string (informational)
     """
     arch = self.arch
-    mcpu, mattr = ARCH_CONFIG[arch]
 
     from extra.assembly.amd.test.test_compare_emulators import get_kernels_from_tinygrad
-    from tinygrad.runtime.support.compiler_amd import HIPCompiler
+    from tinygrad.runtime.support.elf import elf_loader
+    from tinygrad.runtime.support.compiler_amd import HIPCompiler, AMDLLVMCompiler
+    from tinygrad.helpers import AMD_LLVM
 
     kernels, _, _ = get_kernels_from_tinygrad(op_fn)
-    compiler = HIPCompiler(mcpu)
+    # rendered source can be C or llvmir
+    compiler = (AMDLLVMCompiler if AMD_LLVM else HIPCompiler)(get_target(arch))
 
     # First pass: decode all instructions and collect info
     decoded_instrs: list[tuple] = []  # list of (ki, offset, orig_bytes, decoded, our_disasm, decode_ok, decode_err)
     for ki, kernel in enumerate(kernels):
       offset = 0
-      while offset < len(kernel.code):
-        remaining = kernel.code[offset:]
+      code = next((s.content for s in elf_loader(compiler.compile(kernel.src))[1] if s.name == ".text"))
+      while offset < len(code):
+        remaining = code[offset:]
         fmt = detect_format(remaining, arch)
         if fmt is None:
           decoded_instrs.append((ki, offset, None, None, None, False, "no format"))
@@ -243,7 +238,6 @@ class TestTinygradKernelRoundtrip(unittest.TestCase):
   # Fused ops
   def test_fma(self): self._test_kernel_roundtrip(lambda T: (T([1.0, 2.0]) * T([3.0, 4.0]) + T([5.0, 6.0])))
 
-@unittest.skip("RDNA4 decode roundtrip not yet supported")
 class TestTinygradKernelRoundtripRDNA4(TestTinygradKernelRoundtrip): arch = 'rdna4'
 
 @unittest.skip("CDNA decode roundtrip not yet supported")

extra/assembly/amd/test/test_sqtt_examples.py

@@ -5,11 +5,12 @@ from pathlib import Path
 from tinygrad.helpers import DEBUG
 from tinygrad.runtime.autogen import rocprof
 from tinygrad.runtime.support.elf import elf_loader
-from extra.assembly.amd.decode import decode_inst
+from extra.assembly.amd import decode_inst
 from extra.assembly.amd.autogen.rdna3.ins import SOPP
 from extra.assembly.amd.autogen.rdna3.enum import SOPPOp
-from extra.assembly.amd.sqtt import (decode, LAYOUT_HEADER, WAVESTART, WAVEEND, INST, VALUINST, IMMEDIATE, IMMEDIATE_MASK,
-                                     ALUEXEC, VMEMEXEC, PACKET_TYPES, InstOp, print_packets)
+from extra.assembly.amd.sqtt import (decode, LAYOUT_HEADER, WAVESTART, WAVESTART_L4, WAVEEND, INST, INST_L4, VALUINST, IMMEDIATE, IMMEDIATE_MASK,
+                                     ALUEXEC, VMEMEXEC, PACKET_TYPES_L3, PACKET_TYPES_L4, InstOp, InstOpL4, print_packets)
+from extra.assembly.amd.test.helpers import TARGET_TO_ARCH
 
 EXAMPLES_DIR = Path(__file__).parent.parent.parent.parent / "sqtt/examples"
 # INST ops for non-traced SIMDs (excluded from instruction count)
@@ -18,12 +19,13 @@ OTHER_SIMD_OPS = {InstOp.OTHER_LDS_LOAD, InstOp.OTHER_LDS_STORE, InstOp.OTHER_LD
                   InstOp.OTHER_FLAT_STORE_128, InstOp.OTHER_GLOBAL_LOAD, InstOp.OTHER_GLOBAL_LOAD_VADDR,
                   InstOp.OTHER_GLOBAL_STORE_64, InstOp.OTHER_GLOBAL_STORE_96, InstOp.OTHER_GLOBAL_STORE_128,
                   InstOp.OTHER_GLOBAL_STORE_VADDR_128}
+OTHER_SIMD_OPS_L4 = {InstOpL4.OTHER_VMEM, InstOpL4.UNK_60}
 
 # ═══════════════════════════════════════════════════════════════════════════════
 # ROCPROF DECODER
 # ═══════════════════════════════════════════════════════════════════════════════
 
-def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int):
+def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int, target: str):
   """Run rocprof decoder on SQTT blobs, returning raw occupancy and instruction records."""
   image, sections, _ = elf_loader(lib)
   text = next((sh for sh in sections if sh.name == ".text"), None)
@@ -58,6 +60,7 @@ def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int):
           wave_insts.append([(inst.time, inst.stall) for inst in insts])
     return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
 
+  arch = TARGET_TO_ARCH[target]
   @rocprof.rocprof_trace_decoder_isa_callback_t
   def isa_cb(instr_ptr, mem_size_ptr, size_ptr, pc, _):
     offset = pc.address - base
@@ -65,8 +68,9 @@ def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int):
       mem_size_ptr[0] = 0
       return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
     try:
-      inst = decode_inst(image[offset:])
+      inst = decode_inst(image[offset:], arch=arch)
       mem_size_ptr[0] = inst._size()
+    # this could be an error in our decode_inst
     except (ValueError, AssertionError):
       mem_size_ptr[0] = 0
       return rocprof.ROCPROFILER_THREAD_TRACE_DECODER_STATUS_SUCCESS
@@ -88,11 +92,14 @@ def run_rocprof_decoder(blobs: list[bytes], lib: bytes, base: int):
   if t.is_alive(): raise RuntimeError("rocprof decoder timeout")
   return occupancy_records, wave_insts
 
-class TestSQTTExamples(unittest.TestCase):
+class SQTTExamplesTestBase(unittest.TestCase):
+  target: str
+
   @classmethod
   def setUpClass(cls):
+    if cls is SQTTExamplesTestBase: raise unittest.SkipTest("base class")
     cls.examples = {}
-    for pkl_path in sorted(EXAMPLES_DIR.glob("*.pkl")):
+    for pkl_path in sorted((EXAMPLES_DIR/cls.target).glob("*.pkl")):
       with open(pkl_path, "rb") as f:
         data = pickle.load(f)
       sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
@@ -113,18 +120,20 @@ class TestSQTTExamples(unittest.TestCase):
           self.assertIsInstance(packets[0], LAYOUT_HEADER, f"first packet should be LAYOUT_HEADER in {name}")
 
   def test_packet_types_valid(self):
+    all_classes = set(PACKET_TYPES_L3.values()) | set(PACKET_TYPES_L4.values())
     for name, (events, *_) in self.examples.items():
       for i, event in enumerate(events):
         with self.subTest(example=name, event=i):
           for pkt in decode(event.blob):
-            self.assertIn(type(pkt), PACKET_TYPES, f"unknown packet type {type(pkt)} in {name}")
+            # Use isinstance to handle layout-specific subclasses (e.g., WAVESTART_L4)
+            self.assertTrue(any(isinstance(pkt, cls) for cls in all_classes), f"unknown packet type {type(pkt)} in {name}")
 
   def test_wave_lifecycle(self):
     for name, (events, *_) in self.examples.items():
       if "empty" in name: continue
       with self.subTest(example=name):
         all_packets = [p for e in events for p in decode(e.blob)]
-        self.assertGreater(len([p for p in all_packets if isinstance(p, WAVESTART)]), 0, f"no WAVESTART in {name}")
+        self.assertGreater(len([p for p in all_packets if isinstance(p, (WAVESTART, WAVESTART_L4))]), 0, f"no WAVESTART in {name}")
         self.assertGreater(len([p for p in all_packets if isinstance(p, WAVEEND)]), 0, f"no WAVEEND in {name}")
 
   def test_time_monotonic(self):
@@ -139,27 +148,22 @@ class TestSQTTExamples(unittest.TestCase):
       if "gemm" not in name: continue
       with self.subTest(example=name):
         all_packets = [p for e in events for p in decode(e.blob)]
-        self.assertGreater(len([p for p in all_packets if isinstance(p, INST)]), 0, f"no INST packets in {name}")
+        self.assertGreater(len([p for p in all_packets if isinstance(p, (INST, INST_L4))]), 0, f"no INST packets in {name}")
 
+  expected: dict[str, list[int]] = {}  # override in subclasses
   def test_packet_counts(self):
-    expected = {
-      "profile_empty_run_0": [559, 600],
-      "profile_empty_run_1": [517, 570],
-      "profile_gemm_run_0": [1489, 604, 1789, 466, 17570, 407],
-      "profile_gemm_run_1": [1453, 604, 1871, 493, 17827, 460],
-      "profile_plus_run_0": [695, 668],
-      "profile_plus_run_1": [663, 593],
-    }
+    if not self.expected: self.skipTest("no expected packet counts for this target")
     for name, (events, *_) in self.examples.items():
       with self.subTest(example=name):
+        if not self.expected.get(name): continue
         counts = [len(list(decode(e.blob))) for e in events]
-        self.assertEqual(counts, expected[name], f"packet count mismatch in {name}")
+        self.assertEqual(counts, self.expected[name], f"packet count mismatch in {name}")
 
   def test_rocprof_wave_times_match(self):
     """Wave start/end times must match rocprof exactly."""
     for name, (events, lib, base) in self.examples.items():
       with self.subTest(example=name):
-        occupancy, _ = run_rocprof_decoder([e.blob for e in events], lib, base)
+        occupancy, _ = run_rocprof_decoder([e.blob for e in events], lib, base, self.target)
         # extract from rocprof occupancy records
         roc_starts: dict[tuple[int, int, int], int] = {}
         roc_waves: list[tuple[int, int]] = []
@@ -172,7 +176,7 @@ class TestSQTTExamples(unittest.TestCase):
         for event in events:
           wave_starts: dict[tuple[int, int, int], int] = {}
           for p in decode(event.blob):
-            if isinstance(p, WAVESTART): wave_starts[(p.wave, p.simd, p.cu)] = p._time
+            if isinstance(p, (WAVESTART, WAVESTART_L4)): wave_starts[(p.wave, p.simd, p.cu)] = p._time
             elif isinstance(p, WAVEEND) and (key := (p.wave, p.simd, p.cu)) in wave_starts:
               our_waves.append((wave_starts[key], p._time))
         self.assertEqual(sorted(our_waves), sorted(roc_waves), f"wave times mismatch in {name}")
@@ -181,7 +185,7 @@ class TestSQTTExamples(unittest.TestCase):
     """Instruction times must match rocprof exactly (excluding s_endpgm)."""
     for name, (events, lib, base) in self.examples.items():
       with self.subTest(example=name):
-        _, wave_insts = run_rocprof_decoder([e.blob for e in events], lib, base)
+        _, wave_insts = run_rocprof_decoder([e.blob for e in events], lib, base, self.target)
         # skip last inst per wave (s_endpgm) - it needs special handling (time + duration instead of time + stall)
         roc_insts = [time + stall for insts in wave_insts for time, stall in insts[:-1]]
         # extract from our decoder
@@ -189,11 +193,32 @@ class TestSQTTExamples(unittest.TestCase):
         for event in events:
           for p in decode(event.blob):
             if isinstance(p, INST) and p.op not in OTHER_SIMD_OPS: our_insts.append(p._time)
+            elif isinstance(p, INST_L4) and p.op not in OTHER_SIMD_OPS_L4: our_insts.append(p._time)
             elif isinstance(p, VALUINST): our_insts.append(p._time)
             elif isinstance(p, IMMEDIATE): our_insts.append(p._time)
             elif isinstance(p, IMMEDIATE_MASK):
               for _ in range(bin(p.mask).count('1')): our_insts.append(p._time)
         self.assertEqual(sorted(our_insts), sorted(roc_insts), f"instruction times mismatch in {name}")
 
+class TestSQTTExamplesRDNA3(SQTTExamplesTestBase):
+  target = "gfx1100"
+  expected = {
+    "profile_empty_run_0": [1803, 1908, 1928, 1979, 2006, 1912],
+    "profile_empty_run_1": [1803, 1908, 1928, 1979, 2006, 1912],
+    "profile_gemm_run_0": [2531, 1844, 1864, 1915, 1942, 1848, 3074, 1919, 1939, 1990, 2017, 1923, 19026, 1919, 1939, 1990, 2017, 1929],
+    "profile_gemm_run_1": [2554, 1844, 1864, 1915, 1942, 1848, 3084, 1919, 1939, 1990, 2017, 1923, 19010, 1919, 1939, 1990, 2017, 1923],
+    "profile_plus_run_0": [1900, 1908, 1928, 1979, 2006, 1912],
+    "profile_plus_run_1": [1856, 1908, 1928, 1979, 2006, 1912],
+  }
+
+class TestSQTTExamplesRDNA4(SQTTExamplesTestBase): target = "gfx1200"
+# CDNA/MI300 (gfx950) uses a completely different 16-bit header packet format, not the nibble-based format.
+# See decode_tt_header_stream in ghidra/librocprof-trace-decoder.c - it reads 16-bit headers and uses
+# pkt_fmt = header & 0xf to look up packet_class (0x10=2bytes, 0x20=4bytes, 0x30=6bytes, 0x40=8bytes).
+# This is NOT implemented yet - the nibble decoder produces garbage for CDNA data.
+@unittest.skip("CDNA/MI300 uses 16-bit header format, not nibble-based - decoder not implemented")
+class TestSQTTExamplesCDNA(SQTTExamplesTestBase):
+  target = "gfx950"
+
 if __name__ == "__main__":
   unittest.main()

extra/assembly/amd/test/test_sqtt_tables.py

@@ -0,0 +1,170 @@
+"""Tests comparing sqtt.py PACKET_TYPES_L3/L4 against AMD's rocprof-trace-decoder binary."""
+import unittest, struct, ctypes, pickle
+from pathlib import Path
+
+ROCPROF_LIB = Path("/usr/lib/librocprof-trace-decoder.so")
+EXAMPLES_DIR = Path(__file__).parent.parent.parent.parent / "sqtt/examples"
+
+def _find_segment(perms: str):
+  """Find a segment of the loaded library with given permissions (e.g. 'rw-p', 'r--p')."""
+  with open('/proc/self/maps', 'r') as f:
+    for line in f:
+      if 'librocprof-trace-decoder.so' in line and f' {perms} ' in line:
+        parts = line.split()
+        return int(parts[0].split('-')[0], 16), int(parts[2], 16)
+  return None, None
+
+def _read_array(file_offset: int, count: int):
+  """Read an array of uint8 at file_offset from the loaded library."""
+  base, seg_offset = _find_segment('rw-p')
+  if base is None: return None
+  return list((ctypes.c_uint8 * count).from_address(base + (file_offset - seg_offset)))
+
+def _load_lib():
+  if not ROCPROF_LIB.exists(): return False
+  ctypes.CDLL(str(ROCPROF_LIB))
+  return True
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# RDNA EXTRACTION (nibble-based format)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def extract_bit_tables():
+  """Extract bit budget tables. Returns (layout2, layout3, layout4) or None."""
+  if not _load_lib(): return None
+  return _read_array(0x2d220, 32), _read_array(0x2d280, 32), _read_array(0x2d2c0, 32)
+
+def extract_delta_fields():
+  """Extract delta bitfield tables. Returns (layout2, layout3, layout4) dicts mapping type_id -> (lo, hi)."""
+  if not _load_lib(): return None
+  ro_base, ro_offset = _find_segment('r--p')
+  if ro_base is None: return None
+
+  def read_table(file_offset, num_entries):
+    addr = ro_base + (file_offset - ro_offset)
+    data = bytes((ctypes.c_uint8 * (num_entries * 12)).from_address(addr))
+    return {type_id: (lo, hi) for j in range(0, len(data), 12)
+            for type_id, lo, hi in [struct.unpack('<III', data[j:j+12])] if type_id < 32}
+
+  return read_table(0x26800, 24), read_table(0x26dc0, 25), read_table(0x27300, 27)
+
+def extract_packet_encodings():
+  """Extract packet encodings. Returns (L2, L3, L4) dicts mapping type_id -> (mask, value)."""
+  if not _load_lib(): return None
+  rw_base, rw_offset = _find_segment('rw-p')
+  if rw_base is None: return None
+
+  # Read base encodings from registration vector at 0x2d340
+  vec_start = ctypes.c_void_p.from_address(rw_base + (0x2d340 - rw_offset)).value
+  vec_end = ctypes.c_void_p.from_address(rw_base + (0x2d348 - rw_offset)).value
+  base = {}
+  if vec_start and vec_end:
+    for i in range((vec_end - vec_start) // 32):
+      addr = vec_start + i * 32
+      type_id = ctypes.c_uint8.from_address(addr).value
+      pat_start = ctypes.c_void_p.from_address(addr + 8).value
+      pat_end = ctypes.c_void_p.from_address(addr + 16).value
+      if pat_start and pat_end and 0 < (n := pat_end - pat_start) <= 8:
+        pat = list((ctypes.c_uint8 * n).from_address(pat_start))
+        base[type_id] = (sum(1 << j for j in range(n)), sum(b << j for j, b in enumerate(pat)))
+
+  return {**base, 17: (0x7f, 0x51), 25: (0x7f, 0x31)}, base, {**base}  # L2 has overrides
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CDNA EXTRACTION (16-bit header format)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+def extract_cdna_packet_sizes():
+  """Extract CDNA pkt_fmt -> size mapping by running rocprof decoder to populate its hash table."""
+  from extra.assembly.amd.test.test_sqtt_examples import run_rocprof_decoder
+
+  if not (pkl_path := next((EXAMPLES_DIR / "gfx950").glob("*.pkl"), None)): return None
+  with open(pkl_path, "rb") as f: data = pickle.load(f)
+  sqtt_events = [e for e in data if type(e).__name__ == "ProfileSQTTEvent"]
+  prg = next((e for e in data if type(e).__name__ == "ProfileProgramEvent"), None)
+  if not sqtt_events or not prg: return None
+
+  # Run decoder to trigger hash table initialization
+  run_rocprof_decoder([e.blob for e in sqtt_events], prg.lib, prg.base, "gfx950")
+
+  # Extract hash table: head at 0x2d4f0, nodes are 16 bytes (next[8], key[4], value[4])
+  rw_base, rw_offset = _find_segment('rw-p')
+  if not (head := ctypes.c_void_p.from_address(rw_base + (0x2d4f0 - rw_offset)).value if rw_base else None): return None
+
+  pkt_sizes, node, seen = {}, head, set()
+  while node and node not in seen and len(pkt_sizes) < 20:
+    seen.add(node)
+    key, val = ctypes.c_uint32.from_address(node + 8).value, ctypes.c_uint32.from_address(node + 12).value
+    if key < 16 and val in (0x10, 0x20, 0x30, 0x40): pkt_sizes[key] = {0x10: 2, 0x20: 4, 0x30: 6, 0x40: 8}[val]
+    node = ctypes.c_void_p.from_address(node).value
+  return pkt_sizes if len(pkt_sizes) == 16 else None
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# TESTS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class TestSQTTMatchesBinary(unittest.TestCase):
+  def test_bit_counts_match_layout3(self): self._test_bit_counts(3)
+  def test_bit_counts_match_layout4(self): self._test_bit_counts(4)
+  def test_encodings_match_layout3(self): self._test_encodings(3)
+  def test_encodings_match_layout4(self): self._test_encodings(4)
+  def test_delta_fields_match_layout3(self): self._test_delta_fields(3)
+  def test_delta_fields_match_layout4(self): self._test_delta_fields(4)
+
+  def test_cdna_packet_sizes(self):
+    """Extract and verify CDNA pkt_fmt -> size mapping from rocprof's hash table."""
+    if not (EXAMPLES_DIR / "gfx950").exists(): self.skipTest("no CDNA examples")
+    pkt_sizes = extract_cdna_packet_sizes()
+    self.assertIsNotNone(pkt_sizes, "failed to extract CDNA packet sizes")
+    from extra.assembly.amd.sqtt_cdna import CDNA_PKT_SIZES
+    for pkt_fmt, size in CDNA_PKT_SIZES.items():
+      with self.subTest(pkt_fmt=pkt_fmt): self.assertEqual(pkt_sizes.get(pkt_fmt), size)
+
+  def _test_bit_counts(self, layout: int):
+    if not (tables := extract_bit_tables()): self.skipTest("rocprof-trace-decoder not installed")
+    from extra.assembly.amd.sqtt import PACKET_TYPES_L3, PACKET_TYPES_L4
+    for type_id, pkt_cls in {3: PACKET_TYPES_L3, 4: PACKET_TYPES_L4}[layout].items():
+      with self.subTest(packet=pkt_cls.__name__):
+        self.assertEqual(pkt_cls._size_nibbles * 4, tables[layout - 2][type_id])
+
+  def _test_encodings(self, layout: int):
+    if not (encodings := extract_packet_encodings()): self.skipTest("rocprof-trace-decoder not installed")
+    from extra.assembly.amd.sqtt import PACKET_TYPES_L3, PACKET_TYPES_L4
+    for type_id, pkt_cls in {3: PACKET_TYPES_L3, 4: PACKET_TYPES_L4}[layout].items():
+      with self.subTest(packet=pkt_cls.__name__):
+        self.assertEqual((pkt_cls.encoding.mask, pkt_cls.encoding.default), encodings[layout - 2][type_id])
+
+  def _test_delta_fields(self, layout: int):
+    if not (deltas := extract_delta_fields()): self.skipTest("rocprof-trace-decoder not installed")
+    from extra.assembly.amd.sqtt import PACKET_TYPES_L3, PACKET_TYPES_L4
+    for type_id, pkt_cls in {3: PACKET_TYPES_L3, 4: PACKET_TYPES_L4}[layout].items():
+      if type_id not in deltas[layout - 2]: continue
+      delta = getattr(pkt_cls, 'delta', None)
+      actual = (0, 0) if delta is None else (delta.lo, delta.hi + 1)
+      with self.subTest(packet=pkt_cls.__name__): self.assertEqual(actual, deltas[layout - 2][type_id])
+
+if __name__ == "__main__":
+  tables = extract_bit_tables()
+  encodings = extract_packet_encodings()
+  deltas = extract_delta_fields()
+
+  TYPE_NAMES = {1: 'VALUINST', 2: 'VMEMEXEC', 3: 'ALUEXEC', 4: 'IMMEDIATE', 5: 'IMMEDIATE_MASK', 6: 'WAVERDY',
+    7: 'TS_DELTA_S8_W3', 8: 'WAVEEND', 9: 'WAVESTART', 10: 'TS_DELTA_S5_W2', 11: 'WAVEALLOC', 12: 'TS_DELTA_S5_W3',
+    13: 'PERF', 14: 'UTILCTR', 15: 'TS_DELTA_SHORT', 16: 'NOP', 17: 'TS_WAVE_STATE', 18: 'EVENT', 19: 'EVENT_BIG',
+    20: 'REG', 21: 'SNAPSHOT', 22: 'TS_DELTA_OR_MARK', 23: 'LAYOUT_HEADER', 24: 'INST', 25: 'UNK_25'}
+
+  print("L2:", tables[0], "\nL3:", tables[1], "\nL4:", tables[2])
+  if encodings and tables:
+    print(f"\n{'TypeID':>6} {'Name':>18} {'L2 enc':>12} {'L3 enc':>12} {'L4 enc':>12} {'L2':>4} {'L3':>4} {'L4':>4} {'L2 delta':>12} {'L3 delta':>12} {'L4 delta':>12}")
+    print("-" * 140)
+    for type_id in sorted(set(encodings[0]) | set(encodings[1]) | set(encodings[2])):
+      name = TYPE_NAMES.get(type_id, f'UNK_{type_id}')
+      bits = [tables[i][type_id] if type_id < len(tables[i]) else 0 for i in range(3)]
+      enc_strs = [f"0x{encodings[i][type_id][0]:02x}/0x{encodings[i][type_id][1]:02x}" if type_id in encodings[i] else "-" for i in range(3)]
+      delta_strs = [f"[{d[1]-1}:{d[0]}]" if (d := deltas[i].get(type_id, (0, 0)))[1] > d[0] else "-" for i in range(3)]
+      print(f"{type_id:6d} {name:>18} {enc_strs[0]:>12} {enc_strs[1]:>12} {enc_strs[2]:>12} {bits[0]:4d} {bits[1]:4d} {bits[2]:4d} {delta_strs[0]:>12} {delta_strs[1]:>12} {delta_strs[2]:>12}")
+
+  cdna = extract_cdna_packet_sizes()
+  if cdna: print(f"\nCDNA packet sizes: {cdna}")
+
+  unittest.main()

extra/export_model.py

@@ -67,12 +67,11 @@ def export_model_clang(functions:Dict[str,str], statements:Dict[str,Tuple[str,in
   forward_args = ",".join(f"{dtype}{'*' if name not in symbolic_vars.values() else ''} {name}" for name,dtype,_ in (outputs+inputs if wasm else inputs+outputs))
 
   if not wasm:
-    thread_id = 0 # NOTE: export does not support threading, thread_id is always 0
     for name,cl in bufs_to_save.items():
       weight = ''.join(["\\x%02X"%x for x in bytes(to_mv(cl._buf.va_addr, cl._buf.size))])
       cprog.append(f"unsigned char {name}_data[] = \"{weight}\";")
     cprog += [f"{dtype_map[dtype]} {name}[{len}];" if name not in bufs_to_save else f"{dtype_map[dtype]} *{name} = ({dtype_map[dtype]} *){name}_data;" for name,(len,dtype,_key) in bufs.items() if name not in input_names+output_names]
-    cprog += [f"void net({forward_args}) {{"] + [f"{name}({', '.join(args)}, {thread_id});" for (name, args, _global_size, _local_size) in statements] + ["}"]
+    cprog += [f"void net({forward_args}) {{"] + [f"{name}({', '.join(args)});" for (name, args, _global_size, _local_size) in statements] + ["}"]
     return '\n'.join(headers + cprog)
   else:
     if bufs_to_save:

extra/gemm/amd_asm_matmul.py

@@ -6,7 +6,7 @@
 # Workgroup: 128 threads (arranged as 32x4 for coalesced memory access)
 # Inner loop: 8 iterations per K-block, processing 8 columns of A and 8 rows of B
 #
-# Accumulators: 128 vgprs (v[2-117], v[120-124], v[126-129], v[131-133])
+# Accumulators: 128 vgprs (v[2-129])
 
 import numpy as np
 from pathlib import Path
@@ -27,23 +27,20 @@ LDS_B_STRIDE = 0x200  # LDS stride for B tile (512 bytes)
 LDS_BASE_OFFSET = 0x1080  # Base LDS offset for tiles
 ADDR_MASK = 0x3fffff80    # Address alignment mask
 
-# s_waitcnt encodings: wait for memory operations to complete
-WAIT_LGKM = 64519    # wait for LDS/GDS/KMEM (lgkm_cnt=0)
-WAIT_ALL = 0         # wait for everything
-WAIT_VMEM = 1015     # wait for VMEM only (vm_cnt=0, lgkm_cnt=63)
-
 # =============================================================================
-# Named register assignments (VGPRs) - COMPACT LAYOUT
+# Named register assignments (VGPRs)
 # =============================================================================
-V_LANE_ID_MOD8 = 182      # lane_id & 7 (column within 8-wide tile chunk)
-V_OUTPUT_ROW = 171        # output row coordinate
-V_LANE_MOD8_X4 = 174      # V_LANE_ID_MOD8 << 2 (byte offset)
-V_LANE_DIV8_X4 = 175      # (lane_id >> 3) << 2
-V_ADDR_HI_ZERO = 188      # always 0 (for 64-bit address high bits)
-V_LDS_A_BASE = 186        # LDS A-tile base address for inner loop
-V_LDS_B_BASE = 170        # LDS B-tile base address for inner loop
-V_GLOBAL_A_ADDR = 171     # global memory A prefetch address (reuses V_OUTPUT_ROW slot during main loop)
-V_GLOBAL_B_ADDR = 178     # global memory B prefetch address
+V_LANE_ID = 0             # lane_id set on startup
+# Use tile gaps (v146-159) for named regs to minimize max VGPR
+V_LANE_ID_MOD8 = 146      # lane_id & 7
+V_LANE_MOD8_X4 = 147      # (lane_id & 7) << 2
+V_LANE_DIV8_X4 = 150      # ((lane_id >> 3) & 3) << 2
+V_LDS_B_BASE = 151        # LDS B-tile base address for inner loop
+V_LDS_A_BASE = 154        # LDS A-tile base address for inner loop
+V_GLOBAL_A_ADDR = 155     # global memory A prefetch address
+V_GLOBAL_B_ADDR = 158     # global memory B prefetch address
+V_LDS_A_ADDR = 159        # single base register for A stores
+V_LDS_B_ADDR = 162        # single base register for B stores
 
 # LDS tile register destinations - SEPARATE from DATA to avoid overlap
 # A on banks 2-3, B on banks 0-1 to avoid bank conflicts in VOPD
@@ -58,14 +55,11 @@ S_TILE_X = 2              # workgroup_x << 7
 S_TILE_Y = 3              # workgroup_y << 7
 S_DIM_N = 4               # matrix dimension N
 S_LOOP_BOUND = 7          # K-8 (loop termination bound)
-S_A_PTR = (8, 9)          # A matrix base pointer
-S_B_PTR = (10, 11)        # B matrix base pointer
 S_LOOP_CTR = 12           # loop counter (increments by 8)
 S_PREFETCH_FLAG = 13      # prefetch condition flag / row stride in epilogue
 S_WORKGROUP_X = 14        # workgroup_id_x
 S_WORKGROUP_Y = 15        # workgroup_id_y
-# Kernarg load destinations (before copy to working regs)
-S_KERNARG_OUT = (16, 17)  # output pointer from kernarg
+# Kernarg load destinations
 S_KERNARG_A = (20, 21)    # A pointer from kernarg
 S_KERNARG_B = (22, 23)    # B pointer from kernarg
 # Prefetch base pointers (8 pairs each, 16KB/256KB apart)
@@ -100,8 +94,6 @@ FMAC_PAIR_ORDER = [
 
 def derive_fmac_pattern(acc_grid, a_tile_regs=None, b_tile_regs=None):
   """Generate 64 dual FMAC ops from accumulator grid with optimized iteration order."""
-  if a_tile_regs is None: a_tile_regs = V_A_TILE_REGS
-  if b_tile_regs is None: b_tile_regs = V_B_TILE_REGS
   pattern = []
   for idx, (a_pair, b_pair) in enumerate(FMAC_PAIR_ORDER):
     a_even, a_odd = a_pair * 2, a_pair * 2 + 1
@@ -120,14 +112,14 @@ def derive_fmac_pattern(acc_grid, a_tile_regs=None, b_tile_regs=None):
   return pattern
 
 # Derived: 64 dual FMAC operations
-FMAC_PATTERN = derive_fmac_pattern(ACC_GRID)
+FMAC_PATTERN = derive_fmac_pattern(ACC_GRID, V_A_TILE_REGS, V_B_TILE_REGS)
 
 def derive_permute_swaps(acc_grid, out_regs):
   """Derive swap sequence to permute accumulators from FMAC layout to output order.
 
   After FMAC loop: acc_grid[a][b] holds C[a,b]
   Output order: for row_half in 0,1; col_group in 0-3; row_in_group in 0-3; b_off in 0-3
-    -> need C[row_half*4 + row_in_group, col_group*4 + b_off] in descending reg order
+    -> need C[row_half*4 + row_in_group, col_group*4 + b_off] in specified reg order
   """
   def target_ab(i):
     row_half, col_group = i // 64, (i // 16) % 4
@@ -150,45 +142,40 @@ def derive_permute_swaps(acc_grid, out_regs):
   return swaps
 
 # Derived: swap sequence to arrange accumulators for output
-OUT_REGS = list(range(129, 1, -1))
+# Each group of 4 registers is ascending for direct global_store_b128
+OUT_REGS = [r for i in range(32) for r in range(126 - i*4, 130 - i*4)]
 PERMUTE_SWAPS = derive_permute_swaps(ACC_GRID, OUT_REGS)
 
 # =============================================================================
-# LDS tile staging registers - COMPACT LAYOUT
+# LDS tile staging registers
 # =============================================================================
 # DATA regs receive contiguous global prefetch, then write to LDS
 # TILE regs receive scattered LDS loads (ds_load_b64 pairs), then feed FMACs
-# These are SEPARATE - DATA lives during prefetch/store, TILE lives during inner loop
-V_LDS_A_ADDR = 189                            # single base register for A stores (use +512 offsets)
-V_LDS_A_DATA = [155, 172, 173, 154, 159, 176, 177, 158]  # 8 data registers for A prefetch (mod 4: 3,0,1,2,3,0,1,2)
-V_LDS_B_ADDR = 190                            # single base register for B stores (use 16-bit offsets)
-V_LDS_B_DATA = [163, 180, 181, 162, 167, 184, 185, 166]  # 8 data registers for B prefetch (mod 4: 3,0,1,2,3,0,1,2)
+# Contiguous layout with mod4=[3,0,1,2,3,0,1,2] for bank conflict avoidance
+V_LDS_A_DATA = [163, 164, 165, 166, 167, 168, 169, 170]
+V_LDS_B_DATA = [171, 172, 173, 174, 175, 176, 177, 178]
+
+# Initial tile prefetch: (vdst, saddr_lo) - load into A data regs using B prefetch pointers (s[24:31])
+INIT_PREFETCH = [(V_LDS_A_DATA[i], S_PREFETCH_B+2*i) for i in range(4)]
 
 # Global memory prefetch schedule: (vdst1, vdst2, addr_vreg, saddr_lo1, saddr_lo2)
 # First 2 pairs from B prefetch pointers (s[32:39]), next 4 pairs from A prefetch pointers (s[40:55])
 PREFETCH_LOADS = [(V_LDS_A_DATA[4+2*i], V_LDS_A_DATA[4+2*i+1], V_GLOBAL_B_ADDR, S_PREFETCH_B+8+4*i, S_PREFETCH_B+10+4*i) for i in range(2)] + \
                  [(V_LDS_B_DATA[2*(i-2)], V_LDS_B_DATA[2*(i-2)+1], V_GLOBAL_A_ADDR, S_PREFETCH_A+4*(i-2), S_PREFETCH_A+2+4*(i-2)) for i in range(2, 6)]
 
-# Initial tile prefetch: (vdst, saddr_lo) - load into A data regs using B prefetch pointers (s[24:31])
-INIT_PREFETCH = [(V_LDS_A_DATA[i], S_PREFETCH_B+2*i) for i in range(4)]
-
-# Initial tile loads: (vdst, addr_lo) pairs - use temp regs in accumulator gaps
-INIT_TILE_LOADS = [(23,5),(24,9),(25,7),(26,2),(27,11),(28,13),(29,6),(30,8),(31,10),(12,12),(13,14),(3,2),(4,4),(5,8),(6,6),(7,10)]
-
-# A matrix row offset registers (scattered to avoid accumulator conflicts)
-ROW_REGS = [165, 146, 147, 164, 169, 150, 151, 168]  # mod 4: 1,2,3,0,1,2,3,0
-
 # =============================================================================
 # Kernel class
 # =============================================================================
 
 class Kernel:
-  def __init__(self, arch='gfx1100'):
-    self.instructions, self.labels, self.branch_targets, self.arch = [], {}, {}, arch
+  def __init__(self, arch='gfx1100'): self.instructions, self.labels, self.pos, self.arch = [], {}, 0, arch
+  def label(self, name): self.labels[name] = self.pos
 
-  def emit(self, inst): self.instructions.append(inst); return inst
-  def label(self, name): self.labels[name] = len(self.instructions)
-  def branch_to(self, label): self.branch_targets[len(self.instructions) - 1] = label
+  def emit(self, inst, target=None):
+    self.instructions.append(inst)
+    inst._target, inst._pos = target, self.pos
+    self.pos += inst.size()
+    return inst
 
   def waitcnt(self, lgkm=None, vm=None):
     """Wait for memory operations. lgkm=N waits until N lgkm ops remain, vm=N waits until N vmem ops remain."""
@@ -197,16 +184,15 @@ class Kernel:
     self.emit(s_waitcnt(simm16=waitcnt))
 
   def to_asm(self):
-    import re
-    # Instruction stream with labels
-    label_at = {pos: name for name, pos in self.labels.items()}
-    body = []
-    for i, inst in enumerate(self.instructions):
-      if i in label_at: body.append(f'.{label_at[i]}:')
-      asm = inst.disasm()
-      if i in self.branch_targets:
-        asm = re.sub(r'(s_cbranch_\w+|s_branch)\s+\S+', rf'\1 .{self.branch_targets[i]}', asm)
-      body.append('\t' + asm)
+    # Patch branch offsets: simm16 = (target_pos - branch_end_pos) / 4
+    for inst in self.instructions:
+      if inst._target is None: continue
+      offset_dwords = (self.labels[inst._target] - inst._pos - inst.size()) // 4
+      if not -32768 <= offset_dwords <= 32767: raise ValueError(f"branch to '{inst._target}' offset {offset_dwords} exceeds simm16 range")
+      inst.simm16 = offset_dwords
+
+    # TODO: replace this with direct ELF
+    body = ['\t' + inst.disasm() for inst in self.instructions]
 
     # limit wave occupancy by using more LDS
     lds_size = max(LDS_SIZE, 65536//getenv("LIMIT_OCC", 65536))
@@ -218,7 +204,7 @@ class Kernel:
       ('user_sgpr_kernarg_segment_ptr', 1), ('user_sgpr_dispatch_id', 0), ('user_sgpr_private_segment_size', 0),
       ('wavefront_size32', 1), ('uses_dynamic_stack', 0), ('enable_private_segment', 0),
       ('system_sgpr_workgroup_id_x', 1), ('system_sgpr_workgroup_id_y', 1), ('system_sgpr_workgroup_id_z', 0),
-      ('system_sgpr_workgroup_info', 0), ('system_vgpr_workitem_id', 0), ('next_free_vgpr', 192),
+      ('system_sgpr_workgroup_info', 0), ('system_vgpr_workitem_id', 0), ('next_free_vgpr', 179),
       ('next_free_sgpr', 16), ('float_round_mode_32', 0), ('float_round_mode_16_64', 0),
       ('float_denorm_mode_32', 3), ('float_denorm_mode_16_64', 3), ('dx10_clamp', 1), ('ieee_mode', 1),
       ('fp16_overflow', 0), ('workgroup_processor_mode', 0), ('memory_ordered', 1), ('forward_progress', 0),
@@ -236,7 +222,7 @@ class Kernel:
       f'    .group_segment_fixed_size: {lds_size}', '    .kernarg_segment_align: 8',
       '    .kernarg_segment_size: 24', '    .max_flat_workgroup_size: 128', '    .name: kernel',
       '    .private_segment_fixed_size: 0', '    .sgpr_count: 60', '    .symbol: kernel.kd',
-      '    .vgpr_count: 192', '    .wavefront_size: 32', f'amdhsa.target: amdgcn-amd-amdhsa--{self.arch}',
+      '    .vgpr_count: 179', '    .wavefront_size: 32', f'amdhsa.target: amdgcn-amd-amdhsa--{self.arch}',
       'amdhsa.version:', '  - 1', '  - 2', '...', '\t.end_amdgpu_metadata'])
 
 
@@ -251,27 +237,20 @@ def build_kernel(arch='gfx1100'):
   # PROLOGUE: Load kernel arguments, compute tile coordinates and addresses
   # ===========================================================================
   k.emit(s_load_b128(sdata=s[S_KERNARG_A[0]:S_KERNARG_B[1]], sbase=s[0:1], offset=0x0, soffset=NULL))
-  k.emit(s_load_b64(sdata=s[S_KERNARG_OUT[0]:S_KERNARG_OUT[1]], sbase=s[0:1], offset=0x10, soffset=NULL))
+  k.emit(s_load_b64(sdata=s[S_OUT_PTR[0]:S_OUT_PTR[1]], sbase=s[0:1], offset=0x10, soffset=NULL))
   k.emit(s_mov_b32(s[S_DIM_N], MATRIX_DIM))
   k.emit(s_mov_b32(s[S_LOOP_CTR], 0))  # used by LDS swizzle, always 0 for valid workgroups
   k.emit(s_lshl_b32(s[S_TILE_X], s[S_WORKGROUP_X], 7))
   k.emit(s_lshl_b32(s[S_TILE_Y], s[S_WORKGROUP_Y], 7))
 
   # Lane-derived values
-  k.emit(v_and_b32_e32(v[V_LANE_ID_MOD8], 7, v[0]))
-  k.emit(v_lshrrev_b32_e32(v[4], 3, v[0]))
-  k.emit(v_or_b32_e32(v[1], s[S_TILE_X], v[0]))
+  k.emit(v_and_b32_e32(v[V_LANE_ID_MOD8], 7, v[V_LANE_ID]))
+  k.emit(v_lshrrev_b32_e32(v[4], 3, v[V_LANE_ID]))
+  k.emit(v_or_b32_e32(v[1], s[S_TILE_X], v[V_LANE_ID]))
   k.emit(v_or_b32_e32(v[22], s[S_TILE_Y], v[4]))
   k.emit(v_lshlrev_b32_e32(v[V_LANE_MOD8_X4], 2, v[V_LANE_ID_MOD8]))
-  k.emit(v_mov_b32_e32(v[2], 0))  # v[1] always positive, sign extension is 0
-  k.emit(v_lshlrev_b64(v[5:6], 2, v[1:2]))
   k.waitcnt(lgkm=0)
 
-  # Copy pointers to working registers
-  k.emit(s_mov_b64(s[S_OUT_PTR[0]:S_OUT_PTR[1]], s[S_KERNARG_OUT[0]:S_KERNARG_OUT[1]]))
-  k.emit(s_mov_b64(s[S_A_PTR[0]:S_A_PTR[1]], s[S_KERNARG_A[0]:S_KERNARG_A[1]]))
-  k.emit(s_mov_b64(s[S_B_PTR[0]:S_B_PTR[1]], s[S_KERNARG_B[0]:S_KERNARG_B[1]]))
-
   # Compute 8 A and B matrix tile base pointers for prefetch
   k.emit(s_mov_b64(s[S_PREFETCH_B:S_PREFETCH_B+1], s[S_KERNARG_B[0]:S_KERNARG_B[1]]))  # B[0]: no offset
   for i in range(1, 8):  # B: 16KB apart
@@ -283,144 +262,61 @@ def build_kernel(arch='gfx1100'):
     k.emit(s_addc_u32(s[S_PREFETCH_A+i*2+1], s[S_KERNARG_A[1]], 0))
 
   # Global prefetch addresses: B = (tile_x + lane_id) * 4, A = ((tile_y << 12) + (lane_id/8)*4K + lane_id%8) * 4
-  k.emit(v_add_nc_u32_e32(v[V_GLOBAL_B_ADDR], s[S_TILE_X], v[0]))
+  k.emit(v_add_nc_u32_e32(v[V_GLOBAL_B_ADDR], s[S_TILE_X], v[V_LANE_ID]))
   k.emit(v_lshlrev_b32_e32(v[V_GLOBAL_B_ADDR], 2, v[V_GLOBAL_B_ADDR]))
   k.emit(s_lshl_b32(s[19], s[S_TILE_Y], 12))
   k.emit(v_lshl_add_u32(v[V_GLOBAL_A_ADDR], v[4], 12, v[V_LANE_ID_MOD8]))  # (lane_id/8)*4K + lane_id%8
   k.emit(v_add_nc_u32_e32(v[V_GLOBAL_A_ADDR], s[19], v[V_GLOBAL_A_ADDR]))
   k.emit(v_lshlrev_b32_e32(v[V_GLOBAL_A_ADDR], 2, v[V_GLOBAL_A_ADDR]))
 
-  # ===========================================================================
-  # Tile address computation for initial A/B matrix loads
-  # ===========================================================================
-  k.emit(s_lshl_b32(s[S_LOOP_BOUND], s[S_DIM_N], 4))  # row stride = 16*N
-  k.emit(v_mul_lo_u32(v[ROW_REGS[0]], v[22], s[S_DIM_N]))  # A matrix row offsets
-  for i in range(1, 8): k.emit(v_add_nc_u32_e32(v[ROW_REGS[i]], s[S_LOOP_BOUND], v[ROW_REGS[i-1]]))
-
-  def addr64(dst, base_s):  # 64-bit address: v[dst:dst+1] = s[base_s:base_s+1] + v[dst]*4
-    k.emit(v_mov_b32_e32(v[dst+1], 0))  # offset always positive, sign ext = 0
-    k.emit(v_lshlrev_b64(v[dst:dst+1], 2, v[dst:dst+1]))
-    k.emit(v_add_co_u32(v[dst], VCC_LO, s[base_s], v[dst]))
-    k.emit(v_add_co_ci_u32_e32(v[dst+1], s[base_s+1], v[dst+1]))
-
-  def b_addr(dst, mult, tmp=None):  # B address for col + mult*N
-    tmp = tmp if tmp is not None else dst
-    k.emit(v_mad_u32_u24(v[tmp], s[S_DIM_N], mult, v[1]))
-    if tmp != dst:
-      k.emit(v_mov_b32_e32(v[tmp+1], 0))  # offset always positive
-      k.emit(v_lshlrev_b64(v[dst:dst+1], 2, v[tmp:tmp+1]))
-      k.emit(v_add_co_u32(v[dst], VCC_LO, s[S_B_PTR[0]], v[dst]))
-      k.emit(v_add_co_ci_u32_e32(v[dst+1], s[S_B_PTR[1]], v[dst+1]))
-    else: addr64(dst, S_B_PTR[0])
-
-  def a_addr(dst, row_reg, tmp):  # A address for row_reg + lane_id_mod8
-    k.emit(v_add_nc_u32_e32(v[tmp], v[row_reg], v[V_LANE_ID_MOD8]))
-    k.emit(v_mov_b32_e32(v[tmp+1], 0))  # offset always positive
-    k.emit(v_lshlrev_b64(v[dst:dst+1], 2, v[tmp:tmp+1]))
-    k.emit(v_add_co_u32(v[dst], VCC_LO, s[S_A_PTR[0]], v[dst]))
-    k.emit(v_add_co_ci_u32_e32(v[dst+1], s[S_A_PTR[1]], v[dst+1]))
-
-  # Batch 1: B addresses (cols 0-5) and loads
-  k.emit(v_add_co_u32(v[5], VCC_LO, s[S_B_PTR[0]], v[5]))
-  k.emit(v_add_co_ci_u32_e32(v[6], s[S_B_PTR[1]], v[6]))
-  for dst, mult in [(9,1), (7,2), (2,3), (11,4), (13,5)]: b_addr(dst, mult)
-  k.emit(s_clause(simm16=5))  # 6 consecutive global loads
-  for vdst, addr in INIT_TILE_LOADS[:6]: k.emit(global_load_b32(vdst=v[vdst], addr=v[addr:addr+1], saddr=NULL))
-
-  # Batch 2: A addresses (rows 0-4) and loads
-  for dst, ri in [(6,0), (8,1), (10,2), (12,3), (14,4)]:
-    k.emit(v_add_nc_u32_e32(v[dst], v[ROW_REGS[ri]], v[V_LANE_ID_MOD8]))
-    addr64(dst, S_A_PTR[0])
-  k.emit(s_clause(simm16=4))  # 5 consecutive global loads
-  for vdst, addr in INIT_TILE_LOADS[6:11]: k.emit(global_load_b32(vdst=v[vdst], addr=v[addr:addr+1], saddr=NULL))
-
-  # Batch 3: B cols 6-7, A rows 5-7, and loads
-  for dst, mult, tmp in [(2,6,15), (4,7,4)]: b_addr(dst, mult, tmp)
-  for dst, ri, tmp in [(8,5,16), (6,6,18), (10,7,20)]: a_addr(dst, ROW_REGS[ri], tmp)
-  k.emit(s_clause(simm16=4))  # 5 consecutive global loads
-  for vdst, addr in INIT_TILE_LOADS[11:]: k.emit(global_load_b32(vdst=v[vdst], addr=v[addr:addr+1], saddr=NULL))
+  # Do initial loads
+  for vdst, saddr_lo in INIT_PREFETCH:
+    k.emit(global_load_b32(vdst=v[vdst], addr=v[V_GLOBAL_B_ADDR], saddr=s[saddr_lo:saddr_lo+1]))
+  for iter in range(6):
+    vdst1, vdst2, addr, slo1, slo2 = PREFETCH_LOADS[iter]
+    k.emit(global_load_b32(vdst=v[vdst1], addr=v[addr], saddr=s[slo1:slo1+1]))
+    k.emit(global_load_b32(vdst=v[vdst2], addr=v[addr], saddr=s[slo2:slo2+1]))
 
   # ===========================================================================
   # LDS store address computation (bank-conflict-avoiding swizzle)
   # ===========================================================================
   # This section computes LDS store addresses with a swizzle pattern to avoid bank conflicts.
-  # Key outputs:
-  #   v[8]: A-tile initial store base (used only for initial stores with stride64)
-  #   V_LDS_B_ADDR (v145): B-tile store base (used for both initial and main loop)
-  #   V_LANE_DIV8_X4 (v135): (lane_id >> 3) << 2 for epilogue
-  #
   # The swizzle ensures that threads in the same wavefront write to different LDS banks.
   # Formula: swizzled_addr = base + (lane_id & 7) * LDS_A_STRIDE + swizzle_offset
   # where swizzle_offset depends on (lane_id >> 3) to distribute across banks.
-
-  # v[22] = tile_y | (lane_id >> 3) from prologue, used as base for row offsets
-  # Compute 7 row offsets for B-tile rows 1-7 (row 0 computed separately in v[9])
-  k.emit(v_add_nc_u32_e32(v[9], s[S_LOOP_CTR], v[22]))  # row 0 base (S_LOOP_CTR=0)
-  for i in range(7): k.emit(v_or_b32_e32(v[10 + i if i < 2 else 12 + i], 16 * (i + 1), v[22]))  # rows 1-7
-
-  # Extract sign bit of workgroup_x (always 0 for valid workgroups, used for masking)
-  k.emit(s_bfe_i32(s[S_LOOP_BOUND], s[S_WORKGROUP_X], 0x10018))
+  k.emit(v_add_nc_u32_e32(v[9], s[S_LOOP_CTR], v[22]))  # row 0 base
   k.emit(v_and_b32_e32(v[9], ADDR_MASK, v[9]))
-  k.emit(s_lshr_b32(s[S_LOOP_BOUND], s[S_LOOP_BOUND], 25))
-
-  # Compute masked row offsets for bank conflict avoidance pattern
-  # Pattern: v[row] = row_val - (row_val & ADDR_MASK) extracts lower bits
-  k.emit(v_add_nc_u32_e32(v[19], s[S_LOOP_CTR], v[10]))
-  k.emit(v_add_nc_u32_e32(v[8], s[S_LOOP_BOUND], v[1]))  # A-tile base computation
-  for d, r in zip([20, 21, 32, 33, 34, 35], [11, 14, 15, 16, 17, 18]):
-    k.emit(v_add_nc_u32_e32(v[d], s[S_LOOP_CTR], v[r]))
-  k.emit(v_and_b32_e32(v[8], ADDR_MASK, v[8]))
   k.emit(v_sub_nc_u32_e32(v[9], v[22], v[9]))  # row 0 swizzle offset
-  for d, s_ in zip([19, 20, 21, 22, 32, 33, 34], [20, 21, 22, 32, 33, 34, 35]):
-    k.emit(v_and_b32_e32(v[d], ADDR_MASK, v[s_]))
-  k.emit(v_sub_nc_u32_e32(v[8], v[1], v[8]))  # A-tile swizzle
-
-  # Apply swizzle offsets and scale to byte offsets
-  k.emit(v_lshlrev_b32_e32(v[9], 2, v[9]))  # row 0 offset * 4
-  for r, t in zip([10, 11, 14, 15, 16, 17, 18], [19, 20, 21, 22, 32, 33, 34]):
-    k.emit(v_sub_nc_u32_e32(v[r], v[r], v[t]))  # rows 1-7 swizzle
-  k.emit(v_bfe_u32(v[2], v[0], 3, 2))  # v[2] = (lane_id >> 3) & 3
-  k.emit(v_lshlrev_b32_e32(v[8], 2, v[8]))  # A-tile base * 4
-
-  # Compute B-tile base address: LDS_A_STRIDE * (lane_id % 8) + row0_offset
+  k.emit(v_lshlrev_b32_e32(v[9], 2, v[9]))  # * 4
   k.emit(v_mad_u32_u24(v[V_LDS_B_ADDR], LDS_A_STRIDE, v[V_LANE_ID_MOD8], v[9]))
-  # Scale row offsets 1-7 to byte offsets (row 0 already in v[9])
-  for d, r in zip([9, 10, 11, 14, 15, 16, 17], [10, 11, 14, 15, 16, 17, 18]):
-    k.emit(v_lshlrev_b32_e32(v[d], 2, v[r]))
+
+  # For V_LDS_A_BASE and epilogue
+  k.emit(v_bfe_u32(v[2], v[V_LANE_ID], 3, 2))  # v[2] = (lane_id >> 3) & 3
   k.emit(v_lshlrev_b32_e32(v[V_LANE_DIV8_X4], 2, v[2]))
-  k.emit(v_add_nc_u32_e32(v[8], 0x80, v[8]))  # A-tile initial store base + 128
 
-  # Store initial tile data to LDS
-  k.waitcnt(vm=0)
-  for i, (d0, d1) in enumerate([(0,1), (2,3), (4,5), (11,12)]):
-    k.emit(ds_store_2addr_stride64_b32(addr=v[8], data0=v[INIT_TILE_LOADS[d0][0]], data1=v[INIT_TILE_LOADS[d1][0]], offset0=16+i*4, offset1=18+i*4))
-  # B stores: single base with offsets 0,64,128,192,256,320,384,448
-  for i, idx in enumerate([6,7,8,9,10,13,14,15]):
-    offset = i * 64
-    k.emit(ds_store_b32(addr=v[V_LDS_B_ADDR], data0=v[INIT_TILE_LOADS[idx][0]], offset0=offset & 0xFF, offset1=offset >> 8))
-
-  # ===========================================================================
-  # INIT: Compute LDS base addresses, then zero accumulators
-  # ===========================================================================
-  # v[3] = v[1] & 0x7F (lower 7 bits) since S_LOOP_BOUND=0 for valid workgroups
+  # Compute LDS load/store base addresses for inner loop
   k.emit(v_lshlrev_b32_e32(v[2], 4, v[2]))
-  k.emit(v_add_nc_u32_e32(v[3], s[S_LOOP_BOUND], v[1]))
-  k.emit(v_and_b32_e32(v[3], ADDR_MASK, v[3]))
-  k.emit(v_sub_nc_u32_e32(v[3], v[1], v[3]))
+  k.emit(v_and_b32_e32(v[3], 0x7F, v[1]))  # simplified from 3 lines
   k.emit(v_lshl_or_b32(v[V_LDS_B_BASE], v[V_LANE_ID_MOD8], 4, LDS_BASE_OFFSET))
   k.emit(v_lshl_add_u32(v[V_LDS_A_ADDR], v[3], 2, LDS_BASE_OFFSET))
-  k.emit(v_lshlrev_b32_e32(v[3], 2, v[0]))
+  k.emit(v_lshlrev_b32_e32(v[3], 2, v[V_LANE_ID]))
   k.emit(v_and_or_b32(v[V_LDS_A_BASE], 0x180, v[3], v[2]))
 
+  # Do initial stores
+  k.waitcnt(vm=0)
+  for i in range(4):  # A tile: 8 values via 4 stride64 stores
+    k.emit(ds_store_2addr_stride64_b32(addr=v[V_LDS_A_ADDR], data0=v[V_LDS_A_DATA[i*2]], data1=v[V_LDS_A_DATA[i*2+1]], offset0=i*4, offset1=i*4+2))
+  for i in range(8):  # B tile: 8 values via 8 scalar stores with 64-byte spacing
+    offset = i * 64
+    k.emit(ds_store_b32(addr=v[V_LDS_B_ADDR], data0=v[V_LDS_B_DATA[i]], offset0=offset & 0xFF, offset1=offset >> 8))
+
   # Zero all 128 accumulators using VOPD dual moves (64 instructions instead of 128)
   for i in range(0, len(OUT_REGS), 2):
     k.emit(VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32, vdstx=v[OUT_REGS[i]], vdsty=v[OUT_REGS[i+1]], srcx0=0, srcy0=0))
-
   k.emit(s_add_i32(s[S_LOOP_BOUND], s[S_DIM_N], -8))
-  k.emit(s_add_u32(s[S_A_PTR[0]], s[S_A_PTR[0]], 32))
-  k.emit(s_addc_u32(s[S_A_PTR[1]], s[S_A_PTR[1]], 0))
+
   # S_LOOP_CTR is already 0 from prologue initialization
-  k.emit(s_branch(simm16=0)); k.branch_to('LOOP_ENTRY')
+  k.emit(s_branch(), target='LOOP_ENTRY')
 
   # ===========================================================================
   # MAIN GEMM LOOP
@@ -431,19 +327,19 @@ def build_kernel(arch='gfx1100'):
   k.label('LOOP_INC')
   k.emit(s_add_i32(s[S_LOOP_CTR], s[S_LOOP_CTR], 8))
   k.emit(s_cmp_ge_i32(s[S_LOOP_CTR], s[S_DIM_N]))
-  k.emit(s_cbranch_scc1(simm16=0)); k.branch_to('EPILOGUE')
+  k.emit(s_cbranch_scc1(), target='EPILOGUE')
 
   k.label('LOOP_ENTRY')
   k.emit(s_cmp_lt_i32(s[S_LOOP_CTR], s[S_LOOP_BOUND]))
   k.emit(s_cselect_b32(s[S_PREFETCH_FLAG], -1, 0))  # s_cselect doesn't modify SCC
-  k.emit(s_cbranch_scc0(simm16=0)); k.branch_to('SKIP_PREFETCH')  # branch if loop_ctr >= loop_bound
+  k.emit(s_cbranch_scc0(), target='SKIP_PREFETCH')  # branch if loop_ctr >= loop_bound
 
   if not NO_GLOBAL:
     # Advance prefetch pointers (VGPR)
     #k.emit(v_add_nc_u32_e32(v[V_GLOBAL_B_ADDR], 0x20000, v[V_GLOBAL_B_ADDR]))
     #k.emit(v_add_nc_u32_e32(v[V_GLOBAL_A_ADDR], 0x20, v[V_GLOBAL_A_ADDR]))
 
-    # Advance prefetch pointers (SGPRs, 64-bit adds)
+    # Advance prefetch pointers (64-bit adds)
     k.emit(s_clause(simm16=31))
     for i in range(8):
       k.emit(s_add_u32(s[S_PREFETCH_B+i*2], s[S_PREFETCH_B+i*2], 0x20000))
@@ -490,7 +386,7 @@ def build_kernel(arch='gfx1100'):
       k.emit(VOPD(VOPDOp.V_DUAL_FMAC_F32, VOPDOp.V_DUAL_FMAC_F32,
                   vdstx=v[vdst_x], vdsty=v[vdst_y], srcx0=v[ax], vsrcx1=v[bx], srcy0=v[ay], vsrcy1=v[by]))
 
-  # wait for all global stores to finish
+  # wait for all global loads to finish
   # then sync the warp so it's safe to store local
   k.waitcnt(vm=0)
   k.emit(s_barrier())
@@ -507,7 +403,7 @@ def build_kernel(arch='gfx1100'):
       offset = i * 64
       k.emit(ds_store_b32(addr=v[V_LDS_B_ADDR], data0=v[V_LDS_B_DATA[i]], offset0=offset & 0xFF, offset1=offset >> 8))
 
-  k.emit(s_branch(simm16=0)); k.branch_to('LOOP_INC')
+  k.emit(s_branch(), target='LOOP_INC')
 
   # ===========================================================================
   # EPILOGUE: Permute and store results
@@ -518,54 +414,47 @@ def build_kernel(arch='gfx1100'):
   for a, b in PERMUTE_SWAPS:
     k.emit(v_swap_b32_e32(v[a], v[b]))
 
-  # Compute output coordinates: v[V_LANE_ID_MOD8] = col, v[V_OUTPUT_ROW] = row
-  k.emit(VOPD(VOPDOp.V_DUAL_MOV_B32, VOPDOp.V_DUAL_MOV_B32,
-              vdstx=v[149], vdsty=v[150], srcx0=v[V_LANE_MOD8_X4], vsrcx1=v[0], srcy0=v[V_LANE_DIV8_X4], vsrcy1=v[0]))
-  k.emit(v_and_b32_e32(v[0], 0x60, v[0]))
-  k.emit(v_or_b32_e32(v[V_LANE_ID_MOD8], s[S_TILE_X], v[149]))
-  k.emit(v_add_nc_u32_e32(v[0], s[S_TILE_Y], v[0]))
-  k.emit(v_or_b32_e32(v[V_OUTPUT_ROW], v[0], v[150]))
+  # Compute output base coordinates
+  # v[130] = col_base = tile_x + (lane_id & 7) * 4
+  # v[131] = row_base = tile_y + (lane_id & 0x60) + ((lane_id >> 3) & 3) * 4
+  # v[132] = 0 (for 64-bit address high part)
+  k.emit(v_add_nc_u32_e32(v[130], s[S_TILE_X], v[V_LANE_MOD8_X4]))
+  k.emit(v_and_b32_e32(v[131], 0x60, v[V_LANE_ID]))
+  k.emit(v_add_nc_u32_e32(v[131], s[S_TILE_Y], v[131]))
+  k.emit(v_add_nc_u32_e32(v[131], v[V_LANE_DIV8_X4], v[131]))
+  k.emit(v_mov_b32_e32(v[132], 0))
 
-  # Precompute row offsets: v[144-147] for rows 0-3, v[148-151] for rows 16-19
-  for base, row_off in [(144, 0), (148, 16)]:
-    if row_off: k.emit(v_or_b32_e32(v[1], row_off, v[V_OUTPUT_ROW]))
-    k.emit(v_mul_lo_u32(v[base], v[1] if row_off else v[V_OUTPUT_ROW], s[S_DIM_N]))
-    for i in range(3): k.emit(v_add_nc_u32_e32(v[base + 1 + i], s[S_DIM_N], v[base + i]))
+  # Precompute row offsets: v[133-136] for rows 0-3, v[137-140] for rows 16-19
+  for base, row_off in [(133, 0), (137, 16)]:
+    if row_off: k.emit(v_add_nc_u32_e32(v[141], row_off, v[131]))
+    k.emit(v_mul_lo_u32(v[base], v[141] if row_off else v[131], s[S_DIM_N]))
+    for j in range(3): k.emit(v_add_nc_u32_e32(v[base + 1 + j], s[S_DIM_N], v[base + j]))
 
-  k.emit(v_mov_b32_e32(v[V_ADDR_HI_ZERO], 0))
-  k.emit(s_lshl_b32(s[S_PREFETCH_FLAG], s[S_DIM_N], 2))  # row stride in bytes
+  # s[S_PREFETCH_FLAG] = row stride in bytes (N * 4)
+  k.emit(s_lshl_b32(s[S_PREFETCH_FLAG], s[S_DIM_N], 2))
 
   # Store 128 output values as 32 groups of 4 (128-bit stores)
   # Layout: 2 row halves (0-3, 16-19) x 4 col groups x 4 rows = 32 stores of 4 floats
-  epilogue_reserved = {V_LANE_ID_MOD8, V_OUTPUT_ROW, V_LANE_MOD8_X4, V_LANE_DIV8_X4, V_ADDR_HI_ZERO}
-
   for i, (row_half, col_off, row_in_group) in enumerate([(rh, co, ri)
       for rh in range(2) for co in [0, 32, 64, 96] for ri in range(4)]):
     row = row_half * 16 + row_in_group
-    srcs = OUT_REGS[i*4:(i+1)*4]
+    src = OUT_REGS[i*4]  # first reg of ascending group of 4
 
-    # Find temp register for scaled values (must not conflict with reserved regs)
-    tmp = max(srcs) + 5
-    while any(r in epilogue_reserved for r in range(tmp, tmp + 4)): tmp += 1
+    if row_in_group == 0:
+      # First row of group: compute full address
+      if col_off == 0: k.emit(v_mov_b32_e32(v[141], v[130]))
+      else: k.emit(v_add_nc_u32_e32(v[141], col_off, v[130]))
+      row_base = 133 + row if row < 4 else 137 + row - 16
+      k.emit(v_add_nc_u32_e32(v[141], v[row_base], v[141]))
+      k.emit(v_lshlrev_b32_e32(v[141], 2, v[141]))
+      k.emit(v_add_co_u32(v[141], VCC_LO, s[S_OUT_PTR[0]], v[141]))
+      k.emit(v_add_co_ci_u32_e32(v[142], s[S_OUT_PTR[1]], v[132]))
+    else:
+      # Subsequent rows: add stride
+      k.emit(v_add_co_u32(v[141], VCC_LO, s[S_PREFETCH_FLAG], v[141]))
+      k.emit(v_add_co_ci_u32_e32(v[142], v[142], v[132]))
 
-    # Copy values to temp regs for output (alpha=1.0 hardcoded, so just move)
-    for j, src in enumerate(srcs):
-      k.emit(v_mov_b32_e32(v[tmp + j], v[src]))
-
-    # Compute output address
-    if row_in_group == 0:  # first row: compute base address for this column group
-      if col_off == 0: k.emit(v_mov_b32_e32(v[0], v[V_LANE_ID_MOD8]))
-      else: k.emit(v_add_nc_u32_e32(v[0], col_off, v[V_LANE_ID_MOD8]))
-      row_base = 144 + row if row < 4 else 148 + row - 16
-      k.emit(v_add_nc_u32_e32(v[0], v[row_base], v[0]))
-      k.emit(v_lshlrev_b32_e32(v[0], 2, v[0]))
-      k.emit(v_add_co_u32(v[0], VCC_LO, s[S_OUT_PTR[0]], v[0]))
-      k.emit(v_add_co_ci_u32_e32(v[1], s[S_OUT_PTR[1]], v[V_ADDR_HI_ZERO]))
-    else:  # subsequent rows: just add stride
-      k.emit(v_add_co_u32(v[0], VCC_LO, s[S_PREFETCH_FLAG], v[0]))
-      k.emit(v_add_co_ci_u32_e32(v[1], v[1], v[V_ADDR_HI_ZERO]))
-
-    k.emit(global_store_b128(addr=v[0:1], data=v[tmp:tmp+3], saddr=NULL))
+    k.emit(global_store_b128(addr=v[141:142], data=v[src:src+3], saddr=NULL))
 
   k.emit(s_sendmsg(simm16=3))  # DEALLOC_VGPRS
   k.emit(s_endpgm())
@@ -582,7 +471,7 @@ THREADS = 128
 
 def test_matmul():
   dev = Device[Device.DEFAULT]
-  print(f"Device arch: {dev.arch}")
+  print(f"Device arch: {dev.renderer.arch}")
 
   if getenv("STOCK", 0):
     # Load the stock kernel from amd_seb/kernel8_batched_gmem.s
@@ -590,7 +479,7 @@ def test_matmul():
     asm = stock_path.read_text()
     print(f"Loaded stock kernel from {stock_path}")
   else:
-    asm = build_kernel(dev.arch)
+    asm = build_kernel(dev.renderer.arch)
 
   binary = dev.compiler.compile(asm)
   print(f"Compiled! Binary size: {len(binary)} bytes")

extra/gemm/asm/cdna/gemm.py

@@ -0,0 +1,95 @@
+import atexit, functools
+from tinygrad.runtime.support.compiler_amd import HIPCompiler
+from tinygrad import Tensor, Device, dtypes
+from tinygrad.uop.ops import UOp, Ops, KernelInfo, AxisType
+from tinygrad.renderer import Estimates
+from tinygrad.helpers import getenv, all_same, dedup
+from extra.gemm.asm.cdna.asm import build_kernel, GEMM_ARGS
+
+# ** CDNA4 assembly gemm
+
+WORKGROUP_SIZE = 256
+
+def custom_asm_gemm(C:UOp, A:UOp, B:UOp, dname:str, arch:str, wg:int) -> UOp:
+  batch, M, K = A.shape
+  K2, N = B.shape[(1 if B.ndim == 3 else 0):]
+  assert K == K2
+  lidx = UOp.special(WORKGROUP_SIZE, "lidx0")
+  gidx = UOp.special(wg, "gidx0")
+  k = build_kernel(batch, M, N, K, A.dtype.base)
+  sink = UOp.sink(C.base, A.base, B.base, lidx, gidx,
+                  arg=KernelInfo(name=k.name, estimates=Estimates(ops=2*batch*M*N*K, mem=(batch*M*K + K*N + batch*M*N)*2)))
+  binary = HIPCompiler(arch).compile(k.to_asm())
+  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=dname), UOp(Ops.LINEAR, src=(*sink.src, sink)),
+                               UOp(Ops.SOURCE, arg=k.to_text()), UOp(Ops.BINARY, arg=binary)))
+
+counters = {"used":0, "todos":[]}
+def todo(msg:str) -> bool: counters["todos"].append(msg); return False
+atexit.register(lambda: print(f'asm_gemm: {counters["used"]} used, {len(counters["todos"])} not used'))
+
+def can_use_asm_gemm(a:Tensor, b:Tensor) -> bool:
+  if a.dtype != b.dtype: return todo(f"dtypes must match {a.dtype} != {b.dtype}")
+  if a.dtype not in {dtypes.bfloat16, dtypes.float16}: return todo(f"only bfloat16/float16, got {a.dtype}")
+  # only sharding on the batch is tested, others might work too
+  if isinstance(a.device, tuple) and not (a.ndim == 3 and a.uop.axis == 0 and b.uop.axis is None):
+    return todo(f"sharding mismatch a.ndim={a.ndim} a.uop.axis={a.uop.axis} b.uop.axis={b.uop.axis}")
+  batch, M, K = (1, *a.shape) if a.ndim == 2 else a.shape
+  N = b.shape[1]
+  if isinstance(a.device, tuple): batch //= len(a.device)
+  if batch not in {1, 2}: return todo(f"GEMM batch size {batch}")
+  if (key:=(M, N, K)) not in GEMM_ARGS: return todo(f"GEMM shape not supported {key}")
+  return True
+
+# ** UOp gemm to test Tensor.custom_kernel multi and backward correctness on non cdna4
+# note: this can be removed after we have GEMM on mixins
+
+def custom_uop_gemm(C:UOp, A:UOp, B:UOp) -> UOp:
+  M, K = A.shape[0]*A.shape[1], A.shape[2]
+  K2, N = B.shape[(1 if B.ndim == 3 else 0):]
+  assert K == K2
+  m = UOp.range(M, 1, AxisType.LOOP)
+  n = UOp.range(N, 2, AxisType.LOOP)
+  k = UOp.range(K, 0, AxisType.REDUCE)
+  mul = (A.index((m*UOp.const(dtypes.index, K)+k))*B.index((k*UOp.const(dtypes.index, N)+n))).cast(dtypes.float32)
+  red = mul.reduce(k, arg=Ops.ADD, dtype=dtypes.float32).cast(C.dtype.base)
+  store = C.index((m*UOp.const(dtypes.index, N)+n), ptr=True).store(red).end(m, n)
+  return store.sink(arg=KernelInfo(name=f'uop_gemm_{M}_{N}_{K}'))
+
+# ** backward gemm, might use the asm gemm
+
+def custom_gemm_bw(gradient:UOp, kernel:UOp):
+  out, a, b = kernel.src
+  assert all_same([gradient.device, a.device, b.device, out.device])
+  a_t, b_t, g_t = Tensor(a, device=a.device), Tensor(b, device=a.device), Tensor(gradient, device=a.device)
+  grad_a = (g_t @ b_t.T).uop
+  a_T = a_t.transpose(-2, -1)
+  a_T = a_T.reshape(*a_T.shape[:-1], 1, a_T.shape[-1])
+  g_r = g_t.reshape(*g_t.shape[:-2], 1, *g_t.shape[-2:]).transpose(-1, -2)
+  grad_b = (a_T * g_r).sum((-1, 0)).uop
+  return (None, grad_a, grad_b)
+
+# ** main gemm function
+
+def asm_gemm(a:Tensor, b:Tensor) -> Tensor:
+  assert can_use_asm_gemm(a, b), f"{counters['todos'][-1]}"
+  counters["used"] += 1
+  squeeze = a.ndim == 2
+  if squeeze: a = a.unsqueeze(0)
+
+  batch, M, K = a.shape
+  N = b.shape[1]
+  is_multi = isinstance(a.device, tuple)
+
+  if is_multi:
+    out = Tensor(Tensor.empty(batch//len(a.device), M, N, dtype=a.dtype, device=a.device).uop.multi(0), device=a.device)
+  else:
+    out = Tensor.empty(batch, M, N, dtype=a.dtype, device=a.device)
+
+  dname = a.device[0] if is_multi else a.device
+  arch = getattr(Device[dname].renderer, "arch", None)
+  if arch.startswith("gfx950") and getenv("USE_ASM", 1):
+    numWG = GEMM_ARGS[(M, N, K)][0]
+    out = Tensor.custom_kernel(out, a, b, fxn=functools.partial(custom_asm_gemm, dname=dname, wg=numWG, arch=arch), grad_fxn=custom_gemm_bw)[0]
+  else:
+    out = Tensor.custom_kernel(out, a, b, fxn=custom_uop_gemm, grad_fxn=custom_gemm_bw)[0]
+  return out.squeeze(0) if squeeze else out

extra/gemm/asm/cdna/template.s

@@ -1,78 +0,0 @@
-.text
-.section	.text.
-.global	gemm
-.p2align	8
-.type	gemm,@function
-
-gemm:
-INSTRUCTIONS
-
-.section .rodata,"a",@progbits
-.p2align 6, 0x0
-.amdhsa_kernel gemm
-  # basic memory requirements
-  .amdhsa_group_segment_fixed_size 133120
-  .amdhsa_private_segment_fixed_size 0
-  .amdhsa_kernarg_size 28
-  # register usage (RSRC1)
-  .amdhsa_next_free_vgpr 504
-  .amdhsa_next_free_sgpr 96
-  # workgroup / workitem IDs (RSRC2)
-  .amdhsa_system_sgpr_workgroup_id_x 1
-  .amdhsa_system_sgpr_workgroup_id_y 1
-  .amdhsa_system_sgpr_workgroup_id_z 1
-  # user SGPRs, we only specify the kernel args ptr in s[0:1]
-  .amdhsa_user_sgpr_kernarg_segment_ptr 1
-  .amdhsa_user_sgpr_count 2
-  .amdhsa_user_sgpr_kernarg_preload_length 0
-  .amdhsa_user_sgpr_kernarg_preload_offset 0
-  # gfx90a / gfx940 specifics (RSRC3)
-  .amdhsa_accum_offset 248
-  .amdhsa_uses_dynamic_stack 0
-  .amdhsa_tg_split 0
-.end_amdhsa_kernel
-
-.amdgpu_metadata
----
-amdhsa.kernels:
-  - .name: gemm
-    .symbol: gemm.kd
-    .args:
-      - .name: C
-        .address_space: global
-        .offset: 0
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: B
-        .address_space: global
-        .offset: 8
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: A
-        .address_space: global
-        .offset: 16
-        .size: 8
-        .value_kind: global_buffer
-        .value_type: bf16
-      - .name: sz
-        .offset: 24
-        .size: 4
-        .value_kind: by_value
-        .value_type: u32
-    .group_segment_fixed_size: 133120
-    .private_segment_fixed_size: 0
-    .kernarg_segment_align: 8
-    .kernarg_segment_size: 28
-    .max_flat_workgroup_size: 256
-    .sgpr_count: 88
-    .sgpr_spill_count: 0
-    .vgpr_count: 248
-    .vgpr_spill_count: 0
-    .wavefront_size: 64
-amdhsa.version:
-  - 1
-  - 0
-...
-.end_amdgpu_metadata

extra/gemm/asm/cdna/test.py

@@ -1,72 +0,0 @@
-# Run assembly on the AMD runtime and check correctness
-# VIZ=2 to profile
-import pathlib
-from tinygrad import Tensor, Device, dtypes, Context
-from tinygrad.uop.ops import UOp, Ops, KernelInfo
-from tinygrad.helpers import getenv
-
-fp = pathlib.Path(__file__).parent/"gemm.s"
-
-N = getenv("N", 8192)
-THREADS_PER_WG = 256
-NUM_WG = N//THREADS_PER_WG * N//THREADS_PER_WG
-
-assert N % THREADS_PER_WG == 0, "N must be divisible by THREADS_PER_WG"
-
-# ** generate inputs on CPU
-
-scale = 10.0
-
-import torch
-torch.manual_seed(0)
-A = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
-B = (torch.randn(N, N, dtype=torch.float32, device="cpu") / scale).to(torch.bfloat16).contiguous()
-Bt = B.t().contiguous() # transpose B for the asm gemm
-C_torch = A@B
-
-# ** copy buffers to AMD
-
-# input creation and validation run on the copy engine for simpler tracing
-
-def from_torch(t:torch.Tensor) -> Tensor:
-  return Tensor.from_blob(t.data_ptr(), t.shape, dtype=dtypes.bfloat16, device="cpu").to(Device.DEFAULT).realize()
-
-C_tiny = from_torch(A) @ from_torch(B)
-C_asm = Tensor.empty_like(C_tiny)
-
-# ** assembly custom kernel
-
-def custom_asm_gemm(C:UOp, A:UOp, B:UOp) -> UOp:
-  lidx = UOp.special(THREADS_PER_WG, "lidx0")
-  gidx = UOp.special(NUM_WG, "gidx0")
-
-  src = (pathlib.Path(__file__).parent/"template.s").read_text().replace("INSTRUCTIONS", fp.read_text())
-
-  sz = UOp.variable("SZ", 256, 8192)
-
-  sink = UOp.sink(C.base, A.base, B.base, sz, lidx, gidx, arg=KernelInfo(name="gemm"))
-  return UOp(Ops.PROGRAM, src=(sink, UOp(Ops.DEVICE, arg=Device.DEFAULT), UOp(Ops.LINEAR, src=(*sink.src, sink)), UOp(Ops.SOURCE, arg=src)))
-
-C_asm = Tensor.custom_kernel(C_asm, from_torch(A), from_torch(Bt), fxn=custom_asm_gemm)[0]
-
-# ** run gemms
-
-sched = Tensor.schedule(C_tiny, C_asm)
-eis = [si.lower() for si in sched]
-
-with Context(DEBUG=2):
-  for ei in eis:
-    et = ei.run({"SZ":N}, wait=True)
-    print(f"{(N*N*N*2 / et)*1e-12:.2f} REAL TFLOPS")
-
-# ** correctness
-
-import ctypes
-
-def torch_bf16(t:Tensor) -> torch.tensor:
-  asm_out = t.to("cpu").realize().uop.buffer._buf
-  buf = (ctypes.c_uint16*C_asm.uop.size).from_address(asm_out.va_addr)
-  return torch.frombuffer(buf, dtype=torch.bfloat16, count=C_asm.uop.size).reshape(C_asm.shape)
-
-assert torch.allclose(torch_bf16(C_asm), C_torch, rtol=1e-2, atol=1e-3)
-assert torch.allclose(torch_bf16(C_tiny), C_torch, rtol=1e-2, atol=1e-3)

extra/gemm/asm/cdna/test_asm_gemm.py

@@ -0,0 +1,46 @@
+import unittest
+from tinygrad import Tensor, Device, dtypes, Context
+from tinygrad.helpers import getenv
+from extra.gemm.asm.cdna.gemm import asm_gemm
+
+def verify_asm_gemm(batch:int, M:int, N:int, K:int, dtype=dtypes.bfloat16, multi=False) -> None:
+  Tensor.manual_seed(0)
+  a_rand = Tensor.randn((batch, M, K), dtype=dtypes.float).sub(0.5).cast(dtype)
+  b_rand = Tensor.randn((K, N), dtype=dtypes.float).sub(0.5).cast(dtype)
+  with Context(DEBUG=0):
+    Tensor.realize(a_rand, b_rand)
+
+  devs = tuple(f"{Device.DEFAULT}:{i}" for i in range(8)) if multi else None
+
+  a, b = Tensor(a_rand.numpy(), requires_grad=True).cast(dtype), Tensor(b_rand.numpy(), requires_grad=True).cast(dtype)
+  if multi: a, b = a.shard(devs, axis=0), b.shard(devs, axis=None)
+  tst = asm_gemm(a, b)
+  tst.sum().backward()
+  Tensor.realize(tst, a.grad, b.grad)
+
+  a_ref, b_ref = Tensor(a_rand.numpy(), requires_grad=True).cast(dtype), Tensor(b_rand.numpy(), requires_grad=True).cast(dtype)
+  if multi: a_ref, b_ref = a_ref.shard(devs, axis=0), b_ref.shard(devs, axis=None)
+  with Context(ASM_GEMM=0): ref = a_ref @ b_ref
+  ref.sum().backward()
+  Tensor.realize(ref, a_ref.grad, b_ref.grad)
+
+  with Context(DEBUG=0):
+    assert (tst - ref).square().max().float().item() < 1e-6, "forward mismatch"
+    assert (a.grad - a_ref.grad).square().max().float().item() < 1e-3, "grad_a mismatch"
+    assert (b.grad - b_ref.grad).square().max().float().item() < 1e-3, "grad_b mismatch"
+
+class TestGemm(unittest.TestCase):
+  def test_simple(self): verify_asm_gemm(1, N:=getenv("N", 4096), N, N, dtype=dtypes.half)
+
+  def test_gemm1(self): verify_asm_gemm(8, 8192, 4096, 14336, multi=True)
+  def test_gemm2(self): verify_asm_gemm(8, 8192, 128256, 4096, multi=True)
+  def test_gemm3(self): verify_asm_gemm(8, 8192, 14336, 4096, multi=True)
+  def test_gemm4(self): verify_asm_gemm(8, 4096, 14336, 4096, multi=True)
+  def test_gemm5(self): verify_asm_gemm(8, 4096, 4096, 14336, multi=True)
+  def test_gemm6(self): verify_asm_gemm(16, 4096, 4096, 14336, multi=True)
+  def test_gemm_unsupported(self):
+    with self.assertRaisesRegex(AssertionError, "shape not supported"):
+      verify_asm_gemm(8, 8192, 1024, 4096, multi=True)
+
+if __name__ == "__main__":
+  unittest.main()

extra/hcq/hcq_smi.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 
 import argparse, glob, os, time, subprocess, sys
+from tinygrad.helpers import temp
 
 def scan_devs_based_on_lock(prefix:str, args) -> list[str]:
   target_dev = args.pci_bus if 'pci_bus' in args.__dir__() else ""
 
   devs = []
-  for dev in glob.glob(f'/tmp/{prefix}_*.lock'):
-    dev_id = dev[8:-5]
-    if os.path.exists(f"/sys/bus/pci/devices/{dev_id}") and dev_id.startswith(target_dev): devs.append(dev_id)
+  for dev in glob.glob(temp(f'{prefix}_*.lock')):
+    dev_id = dev.split('/')[-1][len(prefix)+1:-5]
+    if dev_id.startswith(target_dev): devs.append(dev_id)
   return devs
 
 def _do_reset_device(pci_bus): os.system(f"sudo sh -c 'echo 1 > /sys/bus/pci/devices/{pci_bus}/reset'")
@@ -53,16 +54,7 @@ def cmd_show_pids(args):
 
   for dev in devs:
     try:
-      pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1]
-      print(f"{dev}: {pid}")
-    except subprocess.CalledProcessError: print(f"{dev}: No processes found using this device")
-
-def cmd_kill_pids(args):
-  devs = scan_devs_based_on_lock(prefix:={"amd":"am", "nv":"nv"}[args.backend], args)
-
-  for dev in devs:
-    try:
-      pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1]
+      pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1]
       print(f"{dev}: {pid}")
     except subprocess.CalledProcessError: print(f"{dev}: No processes found using this device")
 
@@ -74,7 +66,7 @@ def cmd_kill_pids(args):
       if i > 0: time.sleep(0.2)
 
       try:
-        try: pid = subprocess.check_output(['sudo', 'lsof', f'/tmp/{prefix}_{dev}.lock']).decode('utf-8').strip().split('\n')[1].split()[1]
+        try: pid = subprocess.check_output(['sudo', 'lsof', temp(f'{prefix}_{dev}.lock')]).decode('utf-8').strip().split('\n')[1].split()[1]
         except subprocess.CalledProcessError: break
 
         print(f"Killing process {pid} (which uses {dev})")

extra/hip_gpu_driver/hip_ioctl.py

@@ -13,7 +13,7 @@ def get_struct(argp, stype):
 
 def format_struct(s):
   sdats = []
-  for field_name, field_type in s._fields_:
+  for field_name, *_ in s._real_fields_:
     dat = getattr(s, field_name)
     if isinstance(dat, int): sdats.append(f"{field_name}:0x{dat:X}")
     else: sdats.append(f"{field_name}:{dat}")
@@ -53,13 +53,13 @@ def print_aql_queue(read_pointer_address):
   queue_base = read_pointer_address - rptr_offset
   queue = hsa.amd_queue_v2_t.from_address(queue_base)
   print(f"  AQL Queue @ 0x{queue_base:X}:")
-  for field_name, field_type in hsa.amd_queue_v2_t._fields_:
+  for field_name, *_ in hsa.amd_queue_v2_t._real_fields_:
     val = getattr(queue, field_name)
     if isinstance(val, int): print(f"    {field_name}: 0x{val:X}")
     elif hasattr(val, '_length_'):
-      arr_vals = [f"{format_struct(v)}" if hasattr(v, '_fields_') else f"{v:#X}" for v in val]
+      arr_vals = [f"{format_struct(v)}" if hasattr(v, '_real_fields_') else f"{v:#X}" for v in val]
       print(f"    {field_name}: [{', '.join(arr_vals)}]")
-    elif hasattr(val, '_fields_'): print(f"    {field_name}: {format_struct(val)}")
+    elif hasattr(val, '_real_fields_'): print(f"    {field_name}: {format_struct(val)}")
     else: print(f"    {field_name}: {val}")
 
 def ioctls_from_header():

extra/mmapeak/mmapeak.py

@@ -5,7 +5,7 @@ os.environ["AMD_AQL"] = "1"
 
 from tinygrad.device import Device
 from tinygrad.runtime.support.compiler_amd import HIPCompiler
-from tinygrad.runtime.ops_amd import AMDProgram
+from extra.assembly.amd.dsl import Reg, Inst, s, v
 
 NUM_WORKGROUPS = 96
 WAVE_SIZE = 32
@@ -17,84 +17,92 @@ DIRECTIVE = ".amdhsa_wavefront_size32 1"
 
 assemblyTemplate = (pathlib.Path(__file__).parent / "template.s").read_text()
 
-def launchBenchmark(instruction, vgprIndices, dense=True, accum=False, extra=""):
+def repeat(insts:list[Inst], n:int, counter_sreg:Reg) -> bytes:
+  preamble = s_mov_b32(counter_sreg, n).to_bytes()
+  insts_bytes = b"".join([inst.to_bytes() for inst in insts])
+  sub_inst, cmp_inst = s_sub_u32(counter_sreg, counter_sreg, 1), s_cmp_lg_i32(counter_sreg, 0)
+  loop_sz = len(insts_bytes) + sub_inst.size() + cmp_inst.size()
+  branch_inst = s_cbranch_scc1(simm16=-((loop_sz // 4) + 1) & 0xFFFF)
+  return preamble + insts_bytes + sub_inst.to_bytes() + cmp_inst.to_bytes() + branch_inst.to_bytes() + s_endpgm().to_bytes()
+
+def launchBenchmark(instruction, vgprIndices, dense=True, accum=False, **kwargs):
   if accum:
-    instructions = "{} a[0:{}], v[{}:{}], v[{}:{}], 1{}\n".format(instruction, vgprIndices[0],
-                                                                vgprIndices[1], vgprIndices[2],
-                                                                vgprIndices[1], vgprIndices[2], extra)
+    inst = instruction(v[0:vgprIndices[0]], v[vgprIndices[1]:vgprIndices[2]], v[vgprIndices[1]:vgprIndices[2]], 1, acc_cd=1, **kwargs)
   elif dense:
-    instructions = "{} v[0:{}], v[{}:{}], v[{}:{}], 1\n".format(instruction, vgprIndices[0],
-                                                                vgprIndices[1], vgprIndices[2],
-                                                                vgprIndices[1], vgprIndices[2])
+    inst = instruction(v[0:vgprIndices[0]], v[vgprIndices[1]:vgprIndices[2]], v[vgprIndices[1]:vgprIndices[2]], 1)
   else:
-    instructions = "{} v[0:{}], v[{}:{}], v[{}:{}], v{}\n".format(instruction, vgprIndices[0],
-                                                                  vgprIndices[1], vgprIndices[2],
-                                                                  vgprIndices[3], vgprIndices[4],
-                                                                  vgprIndices[5])
-  src = assemblyTemplate.replace("INTERNAL_LOOP", str(INTERNAL_LOOP)).replace("INSTRUCTION", instructions*INSTRUCTIONS_PER_LOOP)
+    inst = instruction(v[0:vgprIndices[0]], v[vgprIndices[1]:vgprIndices[2]], v[vgprIndices[3]:vgprIndices[4]], v[vgprIndices[5]])
+  vgprs:set = set()
+  for n,_ in inst._fields:
+    if isinstance(val:=getattr(inst, n), Reg) and val.offset >= v.offset: vgprs |= {val.offset+i for i in range(val.sz)}
+  inst_bytes = repeat([inst for _ in range(INSTRUCTIONS_PER_LOOP)], n=INTERNAL_LOOP, counter_sreg=s[1])
+  inst_hex = "\n".join("  .byte " + ",".join(f"0x{b:02x}" for b in inst_bytes[i:i+16]) for i in range(0, len(inst_bytes), 16)) + "\n"
+  src = assemblyTemplate.replace("INTERNAL_LOOP", str(INTERNAL_LOOP)).replace("INSTRUCTION", inst_hex).replace("VGPR_COUNT", str(len(vgprs)))
   src = src.replace("DIRECTIVE", DIRECTIVE)
   lib = COMPILER.compile(src)
-  fxn = AMDProgram(DEV, "matmul", lib)
+  fxn = DEV.runtime("matmul", lib)
   elapsed = min([fxn(global_size=(NUM_WORKGROUPS,1,1), local_size=(WAVE_SIZE*NUM_WAVES,1,1), wait=True) for _ in range(2)])
   FLOPs = FLOPS_PER_MATMUL * NUM_WAVES * NUM_WORKGROUPS * INTERNAL_LOOP * INSTRUCTIONS_PER_LOOP
-  print(f"{instruction:<29} : {FLOPs/elapsed/10**12:.2f} T(FL)OPS")
+  print(f"{inst.op_name.lower():<29} : {FLOPs/elapsed/10**12:.2f} T(FL)OPS")
 
 if __name__=="__main__":
-  DEVICENUM = os.getenv("DEVICENUM", "0")
-  try:
-    DEV = Device['AMD:' + DEVICENUM]
-  except:
-    raise RuntimeError("Error while initiating AMD device")
+  DEV = Device[Device.DEFAULT]
+  arch = DEV.renderer.arch
 
-  COMPILER = HIPCompiler(DEV.arch)
-  if DEV.arch in {'gfx1100', 'gfx1103', 'gfx1151'}:
-    if DEV.arch == 'gfx1103': NUM_WORKGROUPS = 8
-    if DEV.arch == 'gfx1151': NUM_WORKGROUPS = 32
-    launchBenchmark("v_wmma_bf16_16x16x16_bf16", (7,8,15))
-    launchBenchmark("v_wmma_f16_16x16x16_f16", (7,8,15))
-    launchBenchmark("v_wmma_f32_16x16x16_bf16", (7,8,15))
-    launchBenchmark("v_wmma_f32_16x16x16_f16", (7,8,15))
-    launchBenchmark("v_wmma_i32_16x16x16_iu4", (7,8,9))
-    launchBenchmark("v_wmma_i32_16x16x16_iu8", (7,8,11))
-  elif DEV.arch == 'gfx1201':
+  COMPILER = HIPCompiler(arch)
+  if arch in {'gfx1100', 'gfx1103', 'gfx1151'}:
+    from extra.assembly.amd.autogen.rdna3.ins import *
+    if arch == 'gfx1103': NUM_WORKGROUPS = 8
+    if arch == 'gfx1151': NUM_WORKGROUPS = 32
+    launchBenchmark(v_wmma_bf16_16x16x16_bf16, (7,8,15))
+    launchBenchmark(v_wmma_f16_16x16x16_f16, (7,8,15))
+    launchBenchmark(v_wmma_f32_16x16x16_bf16, (7,8,15))
+    launchBenchmark(v_wmma_f32_16x16x16_f16, (7,8,15))
+    launchBenchmark(v_wmma_i32_16x16x16_iu4, (7,8,9))
+    launchBenchmark(v_wmma_i32_16x16x16_iu8, (7,8,11))
+  elif arch in {'gfx1200', 'gfx1201'}:
+    from extra.assembly.amd.autogen.rdna4.ins import *
+    # this instruction does not exist in the rdna4 isa, use the co version
+    s_sub_u32 = s_sub_co_u32
     NUM_WORKGROUPS = 64
-    launchBenchmark("v_wmma_bf16_16x16x16_bf16", (3,4,7))
-    launchBenchmark("v_wmma_f16_16x16x16_f16", (3,4,7))
-    launchBenchmark("v_wmma_f32_16x16x16_bf16", (7,8,11))
-    launchBenchmark("v_wmma_f32_16x16x16_f16", (7,8,11))
-    launchBenchmark("v_wmma_i32_16x16x16_iu4", (7,8,8))
-    launchBenchmark("v_wmma_i32_16x16x16_iu8", (7,8,9))
-    launchBenchmark("v_wmma_f32_16x16x16_fp8_fp8", (7,8,9))
-    launchBenchmark("v_wmma_f32_16x16x16_fp8_bf8", (7,8,9))
-    launchBenchmark("v_wmma_f32_16x16x16_bf8_fp8", (7,8,9))
-    launchBenchmark("v_wmma_f32_16x16x16_bf8_bf8", (7,8,9))
+    launchBenchmark(v_wmma_bf16_16x16x16_bf16, (3,4,7))
+    launchBenchmark(v_wmma_f16_16x16x16_f16, (3,4,7))
+    launchBenchmark(v_wmma_f32_16x16x16_bf16, (7,8,11))
+    launchBenchmark(v_wmma_f32_16x16x16_f16, (7,8,11))
+    launchBenchmark(v_wmma_i32_16x16x16_iu4, (7,8,8))
+    launchBenchmark(v_wmma_i32_16x16x16_iu8, (7,8,9))
+    launchBenchmark(v_wmma_f32_16x16x16_fp8_fp8, (7,8,9))
+    launchBenchmark(v_wmma_f32_16x16x16_fp8_bf8, (7,8,9))
+    launchBenchmark(v_wmma_f32_16x16x16_bf8_fp8, (7,8,9))
+    launchBenchmark(v_wmma_f32_16x16x16_bf8_bf8, (7,8,9))
     FLOPS_PER_MATMUL = 16*16*32*2
-    launchBenchmark("v_wmma_i32_16X16X32_iu4", (7,8,9))
-    launchBenchmark("v_swmmac_f32_16x16x32_f16", (7,8,11,12,19,20), False)
-    launchBenchmark("v_swmmac_f32_16x16x32_bf16", (7,8,11,12,19,20), False)
-    launchBenchmark("v_swmmac_f16_16x16x32_f16", (3,4,7,8,15,16), False)
-    launchBenchmark("v_swmmac_bf16_16x16x32_bf16", (3,4,7,8,15,16), False)
-    launchBenchmark("v_swmmac_i32_16x16x32_iu8", (7,8,9,10,13,14), False)
-    launchBenchmark("v_swmmac_i32_16x16x32_iu4", (7,8,8,9,10,11), False)
-    launchBenchmark("v_swmmac_f32_16x16x32_fp8_fp8", (7,8,9,10,13,14), False)
-    launchBenchmark("v_swmmac_f32_16x16x32_fp8_bf8", (7,8,9,10,13,14), False)
-    launchBenchmark("v_swmmac_f32_16x16x32_bf8_fp8", (7,8,9,10,13,14), False)
-    launchBenchmark("v_swmmac_f32_16x16x32_bf8_bf8", (7,8,9,10,13,14), False)
+    launchBenchmark(v_wmma_i32_16x16x32_iu4, (7,8,9))
+    launchBenchmark(v_swmmac_f32_16x16x32_f16, (7,8,11,12,19,20), False)
+    launchBenchmark(v_swmmac_f32_16x16x32_bf16, (7,8,11,12,19,20), False)
+    launchBenchmark(v_swmmac_f16_16x16x32_f16, (3,4,7,8,15,16), False)
+    launchBenchmark(v_swmmac_bf16_16x16x32_bf16, (3,4,7,8,15,16), False)
+    launchBenchmark(v_swmmac_i32_16x16x32_iu8, (7,8,9,10,13,14), False)
+    launchBenchmark(v_swmmac_i32_16x16x32_iu4, (7,8,8,9,10,11), False)
+    launchBenchmark(v_swmmac_f32_16x16x32_fp8_fp8, (7,8,9,10,13,14), False)
+    launchBenchmark(v_swmmac_f32_16x16x32_fp8_bf8, (7,8,9,10,13,14), False)
+    launchBenchmark(v_swmmac_f32_16x16x32_bf8_fp8, (7,8,9,10,13,14), False)
+    launchBenchmark(v_swmmac_f32_16x16x32_bf8_bf8, (7,8,9,10,13,14), False)
     FLOPS_PER_MATMUL = 16*16*64*2
-    launchBenchmark("v_swmmac_i32_16x16x64_iu4", (7,8,9,10,13,14), False)
-  elif DEV.arch == 'gfx950':
+    launchBenchmark(v_swmmac_i32_16x16x64_iu4, (7,8,9,10,13,14), False)
+  elif arch == 'gfx950':
+    from extra.assembly.amd.autogen.cdna.ins import *
     DIRECTIVE = ".amdhsa_accum_offset 4"
     NUM_WORKGROUPS = 256
     WAVE_SIZE = 64
     NUM_WAVES = 4
-    launchBenchmark("v_mfma_f32_16x16x16_f16", (3,0,1), accum=True)
-    launchBenchmark("v_mfma_f32_16x16x16_bf16", (3,0,1), accum=True)
+    launchBenchmark(v_mfma_f32_16x16x16_f16, (3,0,1), accum=True)
+    launchBenchmark(v_mfma_f32_16x16x16_bf16, (3,0,1), accum=True)
     FLOPS_PER_MATMUL = 16*16*32*2
-    launchBenchmark("v_mfma_f32_16x16x32_f16", (3,0,3), accum=True)
-    launchBenchmark("v_mfma_f32_16x16x32_bf16", (3,0,3), accum=True)
+    launchBenchmark(v_mfma_f32_16x16x32_f16, (3,0,3), accum=True)
+    launchBenchmark(v_mfma_f32_16x16x32_bf16, (3,0,3), accum=True)
     FLOPS_PER_MATMUL = 16*16*128*2
-    launchBenchmark("v_mfma_f32_16x16x128_f8f6f4", (3,0,7), accum=True) # fp8
-    launchBenchmark("v_mfma_f32_16x16x128_f8f6f4", (3,0,5), accum=True, extra=", cbsz:2 blgp:2") # fp6
-    launchBenchmark("v_mfma_f32_16x16x128_f8f6f4", (3,0,3), accum=True, extra=", cbsz:4 blgp:4") # fp4
+    launchBenchmark(v_mfma_f32_16x16x128_f8f6f4, (3,0,7), accum=True) # fp8
+    launchBenchmark(v_mfma_f32_16x16x128_f8f6f4, (3,0,5), accum=True, cbsz=2, blgp=2) # fp6
+    launchBenchmark(v_mfma_f32_16x16x128_f8f6f4, (3,0,3), accum=True, cbsz=4, blgp=4) # fp4
   else:
-    raise RuntimeError(f"arch {DEV.arch} not supported.")
+    raise RuntimeError(f"arch {arch} not supported.")

extra/mmapeak/template.s

@@ -3,20 +3,13 @@
     .p2align 8
     .type matmul,@function
 matmul:
-  s_mov_b32 s1, INTERNAL_LOOP
-  s_mov_b32 s2, 0
-  inner_loop:
     INSTRUCTION
-    s_sub_u32 s1, s1, 1
-    s_cmp_lg_i32 s1, s2
-    s_cbranch_scc1 inner_loop
-  s_endpgm
 
 .rodata
 .p2align 6
 .amdhsa_kernel matmul
-  .amdhsa_next_free_vgpr .amdgcn.next_free_vgpr
-  .amdhsa_next_free_sgpr .amdgcn.next_free_sgpr
+  .amdhsa_next_free_vgpr VGPR_COUNT
+  .amdhsa_next_free_sgpr 3
   DIRECTIVE
 .end_amdhsa_kernel
 
@@ -37,4 +30,4 @@ amdhsa.kernels:
     .vgpr_count: 32
     .max_flat_workgroup_size: 1024
 ...
-.end_amdgpu_metadata
+.end_amdgpu_metadata

extra/nv_gpu_driver/nv_ioctl.py

@@ -1,6 +1,7 @@
 # type: ignore
 import ctypes, ctypes.util, struct, platform, pathlib, re, time, os, signal
-from tinygrad.helpers import from_mv, to_mv, getenv, init_c_struct_t
+from tinygrad.helpers import from_mv, to_mv, getenv
+from tinygrad.runtime.support.c import init_c_struct_t
 from hexdump import hexdump
 start = time.perf_counter()
 
@@ -10,18 +11,21 @@ processor = platform.processor()
 IOCTL_SYSCALL = {"aarch64": 0x1d, "x86_64":16}[processor]
 MMAP_SYSCALL = {"aarch64": 0xde, "x86_64":0x09}[processor]
 
+IOCTL_PRINT = getenv("IOCTL_PRINT", getenv("IOCTL", 0))
+GRAB_PMA = getenv("GRAB_PMA", 0)
+
 def get_struct(argp, stype):
   return ctypes.cast(ctypes.c_void_p(argp), ctypes.POINTER(stype)).contents
 
 def dump_struct(st):
-  if getenv("IOCTL", 0) == 0: return
+  if IOCTL_PRINT == 0: return
   print("\t", st.__class__.__name__, end=" { ")
-  for v in type(st)._fields_: print(f"{v[0]}={getattr(st, v[0])}", end=" ")
+  for v in type(st)._real_fields_: print(f"{v[0]}={getattr(st, v[0])}", end=" ")
   print("}")
 
 def format_struct(s):
   sdats = []
-  for field in s._fields_:
+  for field in s._real_fields_:
     dat = getattr(s, field[0])
     if isinstance(dat, int): sdats.append(f"{field[0]}:0x{dat:X}")
     else: sdats.append(f"{field[0]}:{dat}")
@@ -58,6 +62,29 @@ def install_hook(c_function, python_function):
   return orig_func
 
 # *** ioctl lib end ***
+
+# PMA buffer tracking for raw PC sampling data (only when GRAB_PMA is enabled)
+pma_mem_handle = 0  # hMemPmaBuffer from ALLOC_PMA_STREAM
+pma_buffer_size = 0
+pma_buffer_va = 0   # actual mapped VA (found via /proc/self/maps)
+pma_get_offset = 0  # current read offset in ring buffer
+pma_pending_map = False  # flag to check for new mapping on next ioctl
+pma_maps_before = set()  # mappings before MAP_MEMORY
+pma_raw_dumps: list[bytes] = []  # raw PMA buffer dumps
+
+def get_pma_raw_dumps() -> list[bytes]: return pma_raw_dumps
+def clear_pma_raw_dumps(): pma_raw_dumps.clear()
+
+def get_proc_maps():
+  """Read current process memory mappings as set of (start, end) tuples."""
+  result = set()
+  with open("/proc/self/maps", "r") as f:
+    for line in f:
+      addr_range = line.split()[0]
+      start, end = addr_range.split("-")
+      result.add((int(start, 16), int(end, 16)))
+  return result
+
 from tinygrad.runtime.autogen import nv_570 as nv_gpu
 nvescs = {getattr(nv_gpu, x):x for x in dir(nv_gpu) if x.startswith("NV_ESC")}
 nvcmds = {getattr(nv_gpu, x):(x, getattr(nv_gpu, "struct_"+x+"_PARAMS", getattr(nv_gpu, "struct_"+x.replace("_CMD_", "_")+"_PARAMS", None))) for x in dir(nv_gpu) if \
@@ -69,6 +96,7 @@ def get_classes():
                    "NV20_SUBDEVICE_0"}
   for nm,val in nv_gpu.__dict__.items():
     if not isinstance(val, int): continue
+    if nm.endswith("PARAMETERS_MESSAGE_ID"): continue
     if 0x3000 < val < 0xffff: res[val] = nm
     if nm in known_classes: res[val] = nm
   return res
@@ -80,37 +108,92 @@ global_ioctl_id = 0
 gpus_user_modes = []
 gpus_mmio = []
 gpus_fifo = []
+offset_load = 0
 
 @ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulong, ctypes.c_void_p)
 def ioctl(fd, request, argp):
   global global_ioctl_id, gpus_user_modes, gpus_mmio
+  global pma_mem_handle, pma_buffer_size, pma_buffer_va, pma_get_offset, pma_pending_map, pma_maps_before
   global_ioctl_id += 1
+
+  # Check for new PMA buffer mapping from previous MAP_MEMORY call (only when GRAB_PMA is enabled)
+  if GRAB_PMA and pma_pending_map:
+    pma_pending_map = False
+    new_maps = get_proc_maps()
+    for start, end in new_maps - pma_maps_before:
+      if end - start == pma_buffer_size:
+        pma_buffer_va = start
+        if IOCTL_PRINT >= 1: print(f"\t  PMA buffer mapped at CPU VA=0x{pma_buffer_va:x}")
+        break
+
   st = time.perf_counter()
   ret = libc.syscall(IOCTL_SYSCALL, ctypes.c_int(fd), ctypes.c_ulong(request), ctypes.c_void_p(argp))
   et = time.perf_counter()-st
   fn = os.readlink(f"/proc/self/fd/{fd}")
-  #print(f"ioctl {request:8x} {fn:20s}")
+
   idir, size, itype, nr = (request>>30), (request>>16)&0x3FFF, (request>>8)&0xFF, request&0xFF
-  if getenv("IOCTL", 0) >= 1: print(f"#{global_ioctl_id}: ", end="")
+  if IOCTL_PRINT >= 1: print(f"#{global_ioctl_id}: ", end="")
   if itype == ord(nv_gpu.NV_IOCTL_MAGIC):
     if nr == nv_gpu.NV_ESC_RM_CONTROL:
       s = get_struct(argp, nv_gpu.NVOS54_PARAMETERS)
       if s.cmd in nvcmds:
         name, struc = nvcmds[s.cmd]
-        if getenv("IOCTL", 0) >= 1:
+        if IOCTL_PRINT >= 1:
           print(f"NV_ESC_RM_CONTROL    cmd={name:30s} hClient={s.hClient}, hObject={s.hObject}, flags={s.flags}, params={s.params}, paramsSize={s.paramsSize}, status={s.status}")
 
         if struc is not None: dump_struct(get_struct(s.params, struc))
         elif hasattr(nv_gpu, name+"_PARAMS"): dump_struct(get_struct(argp, getattr(nv_gpu, name+"_PARAMS")))
         elif name == "NVA06C_CTRL_CMD_GPFIFO_SCHEDULE": dump_struct(get_struct(argp, nv_gpu.NVA06C_CTRL_GPFIFO_SCHEDULE_PARAMS))
         elif name == "NV83DE_CTRL_CMD_GET_MAPPINGS": dump_struct(get_struct(s.params, nv_gpu.NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS))
+        elif name == "NVB0CC_CTRL_CMD_SET_HS_CREDITS":
+          hs_params = get_struct(s.params, nv_gpu.NVB0CC_CTRL_SET_HS_CREDITS_PARAMS)
+          dump_struct(hs_params)
+          if IOCTL_PRINT >= 2:
+            for i in range(hs_params.numEntries):
+              print(f"\t\t", end="")
+              dump_struct(hs_params.creditInfo[i])
+
+        # PMA buffer tracking (only when GRAB_PMA is enabled)
+        if GRAB_PMA and name == "NVB0CC_CTRL_CMD_ALLOC_PMA_STREAM":
+          pma_params = get_struct(s.params, nv_gpu.struct_NVB0CC_CTRL_ALLOC_PMA_STREAM_PARAMS)
+          pma_mem_handle = pma_params.hMemPmaBuffer
+          pma_buffer_size = pma_params.pmaBufferSize
+          pma_get_offset = 0  # Reset read offset for new stream
+          if IOCTL_PRINT >= 1: print(f"\t  PMA buffer: hMem=0x{pma_mem_handle:x} size={pma_buffer_size}")
+        if GRAB_PMA and name == "NVB0CC_CTRL_CMD_PMA_STREAM_UPDATE_GET_PUT":
+          pma_update = get_struct(s.params, nv_gpu.struct_NVB0CC_CTRL_PMA_STREAM_UPDATE_GET_PUT_PARAMS)
+          if pma_update.bytesAvailable > 0 and pma_buffer_va and pma_buffer_size > 0:
+            avail = pma_update.bytesAvailable
+            read_offset = pma_get_offset
+            # Handle ring buffer wrap-around
+            if pma_get_offset + avail <= pma_buffer_size:
+              pma_data = bytes(to_mv(pma_buffer_va + pma_get_offset, avail))
+            else:
+              # Wrap around: read to end, then from start
+              first_part = pma_buffer_size - pma_get_offset
+              second_part = avail - first_part
+              pma_data = bytes(to_mv(pma_buffer_va + pma_get_offset, first_part)) + bytes(to_mv(pma_buffer_va, second_part))
+            pma_raw_dumps.append(pma_data)
+            pma_get_offset = (pma_get_offset + avail) % pma_buffer_size
+            if IOCTL_PRINT >= 2:
+              print(f"\t  PMA data: {avail} bytes from offset=0x{read_offset:x}, new offset=0x{pma_get_offset:x}")
+              hexdump(pma_data)
+
+        # Dump regOps for EXEC_REG_OPS when IOCTL >= 3
+        if name == "NVB0CC_CTRL_CMD_EXEC_REG_OPS" and struc is not None and IOCTL_PRINT >= 3:
+          reg_params = get_struct(s.params, struc)
+          for i in range(reg_params.regOpCount):
+            print(f"\t\t", end="")
+            dump_struct(reg_params.regOps[i])
+            # val = (op.regValueHi << 32) | op.regValueLo
+            # print(f"\t  regOps[{i:3d}]: op={op.regOp} type={op.regType} status={op.regStatus} offset=0x{op.regOffset:08x} value=0x{val:016x}")
       else:
-        if getenv("IOCTL", 0) >= 1: print("unhandled cmd", hex(s.cmd))
+        if IOCTL_PRINT >= 1: print("unhandled cmd", hex(s.cmd))
       # format_struct(s)
       # print(f"{(st-start)*1000:7.2f} ms +{et*1000.:7.2f} ms : {ret:2d} = {name:40s}", ' '.join(format_struct(s)))
     elif nr == nv_gpu.NV_ESC_RM_ALLOC:
       s = get_struct(argp, nv_gpu.NVOS21_PARAMETERS)
-      if getenv("IOCTL", 0) >= 1: print(f"NV_ESC_RM_ALLOC    hClass={nvclasses.get(s.hClass, f'unk=0x{s.hClass:X}'):30s}, hRoot={s.hRoot}, hObjectParent={s.hObjectParent}, pAllocParms={s.pAllocParms}, hObjectNew={s.hObjectNew} status={s.status}")
+      if IOCTL_PRINT >= 1: print(f"NV_ESC_RM_ALLOC    hClass={nvclasses.get(s.hClass, f'unk=0x{s.hClass:X}'):30s}, hRoot={s.hRoot}, hObjectParent={s.hObjectParent}, pAllocParms={s.pAllocParms}, hObjectNew={s.hObjectNew} status={s.status}")
       if s.pAllocParms is not None:
         if s.hClass == nv_gpu.NV01_DEVICE_0: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV0080_ALLOC_PARAMETERS))
         if s.hClass == nv_gpu.FERMI_VASPACE_A: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_VASPACE_ALLOCATION_PARAMETERS))
@@ -118,7 +201,8 @@ def ioctl(fd, request, argp):
         if s.hClass == nv_gpu.NV1_MEMORY_USER: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_MEMORY_ALLOCATION_PARAMS))
         if s.hClass == nv_gpu.NV1_MEMORY_SYSTEM: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV_MEMORY_ALLOCATION_PARAMS))
         if s.hClass == nv_gpu.GT200_DEBUGGER: dump_struct(get_struct(s.pAllocParms, nv_gpu.NV83DE_ALLOC_PARAMETERS))
-        if s.hClass == nv_gpu.AMPERE_CHANNEL_GPFIFO_A:
+        if s.hClass == nv_gpu.MAXWELL_PROFILER_DEVICE: dump_struct(get_struct(s.pAllocParms, nv_gpu.NVB2CC_ALLOC_PARAMETERS))
+        if s.hClass in {nv_gpu.AMPERE_CHANNEL_GPFIFO_A, nv_gpu.BLACKWELL_CHANNEL_GPFIFO_A}:
           sx = get_struct(s.pAllocParms, nv_gpu.NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS)
           dump_struct(sx)
           gpus_fifo.append((sx.gpFifoOffset, sx.gpFifoEntries))
@@ -126,31 +210,35 @@ def ioctl(fd, request, argp):
       if s.hClass == nv_gpu.TURING_USERMODE_A: gpus_user_modes.append(s.hObjectNew)
     elif nr == nv_gpu.NV_ESC_RM_MAP_MEMORY:
       # nv_ioctl_nvos33_parameters_with_fd
-      if getenv("IOCTL", 0) >= 1:
-        s = get_struct(argp, nv_gpu.NVOS33_PARAMETERS)
+      s = get_struct(argp, nv_gpu.NVOS33_PARAMETERS)
+      if IOCTL_PRINT >= 1:
         print(f"NV_ESC_RM_MAP_MEMORY   hClient={s.hClient}, hDevice={s.hDevice}, hMemory={s.hMemory}, length={s.length} flags={s.flags} pLinearAddress={s.pLinearAddress}")
+      # Track PMA buffer mapping - save maps now, check for new mapping on next ioctl (after mmap happens)
+      if GRAB_PMA and pma_mem_handle and s.hMemory == pma_mem_handle:
+        pma_maps_before = get_proc_maps()
+        pma_pending_map = True
     elif nr == nv_gpu.NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO:
-      if getenv("IOCTL", 0) >= 1:
+      if IOCTL_PRINT >= 1:
         s = get_struct(argp, nv_gpu.NVOS56_PARAMETERS)
         print(f"NV_ESC_RM_UPDATE_DEVICE_MAPPING_INFO   hClient={s.hClient}, hDevice={s.hDevice}, hMemory={s.hMemory}, pOldCpuAddress={s.pOldCpuAddress} pNewCpuAddress={s.pNewCpuAddress} status={s.status}")
     elif nr == nv_gpu.NV_ESC_RM_ALLOC_MEMORY:
-      if getenv("IOCTL", 0) >= 1:
+      if IOCTL_PRINT >= 1:
         s = get_struct(argp, nv_gpu.nv_ioctl_nvos02_parameters_with_fd)
         print(f"NV_ESC_RM_ALLOC_MEMORY  fd={s.fd}, hRoot={s.params.hRoot}, hObjectParent={s.params.hObjectParent}, hObjectNew={s.params.hObjectNew}, hClass={s.params.hClass}, flags={s.params.flags}, pMemory={s.params.pMemory}, limit={s.params.limit}, status={s.params.status}")
     elif nr == nv_gpu.NV_ESC_ALLOC_OS_EVENT:
-      if getenv("IOCTL", 0) >= 1:
+      if IOCTL_PRINT >= 1:
         s = get_struct(argp, nv_gpu.nv_ioctl_alloc_os_event_t)
         print(f"NV_ESC_ALLOC_OS_EVENT  hClient={s.hClient} hDevice={s.hDevice} fd={s.fd} Status={s.Status}")
     elif nr == nv_gpu.NV_ESC_REGISTER_FD:
-      if getenv("IOCTL", 0) >= 1:
+      if IOCTL_PRINT >= 1:
         s = get_struct(argp, nv_gpu.nv_ioctl_register_fd_t)
         print(f"NV_ESC_REGISTER_FD  fd={s.ctl_fd}")
     elif nr in nvescs:
-      if getenv("IOCTL", 0) >= 1: print(nvescs[nr])
+      if IOCTL_PRINT >= 1: print(nvescs[nr])
     else:
-      if getenv("IOCTL", 0) >= 1: print("unhandled NR", nr)
+      if IOCTL_PRINT >= 1: print("unhandled NR", nr)
   elif fn.endswith("nvidia-uvm"):
-    if getenv("IOCTL", 0) >= 1:
+    if IOCTL_PRINT >= 1:
       print(f"{nvuvms.get(request, f'UVM UNKNOWN {request=}')}")
       if nvuvms.get(request) is not None: dump_struct(get_struct(argp, getattr(nv_gpu, nvuvms.get(request)+"_PARAMS")))
       if nvuvms.get(request) == "UVM_MAP_EXTERNAL_ALLOCATION":
@@ -159,7 +247,7 @@ def ioctl(fd, request, argp):
           print("perGpuAttributes[{i}] = ", end="")
           dump_struct(st.perGpuAttributes[i])
 
-  if getenv("IOCTL") >= 2: print("ioctl", f"{idir=} {size=} {itype=} {nr=} {fd=} {ret=}", fn)
+  if IOCTL_PRINT >= 2: print("ioctl", f"{idir=} {size=} {itype=} {nr=} {fd=} {ret=}", fn)
   return ret
 
 @ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_long)
@@ -172,14 +260,14 @@ def _mmap(addr, length, prot, flags, fd, offset):
   return ret
 
 install_hook(libc.ioctl, ioctl)
-if getenv("IOCTL") >= 3: orig_mmap_mv = install_hook(libc.mmap, _mmap)
+if getenv("IOCTL") >= 4: orig_mmap_mv = install_hook(libc.mmap, _mmap)
 
 import collections
 old_gpputs = collections.defaultdict(int)
 def _dump_gpfifo(mark):
   launches = []
 
-  # print("_dump_gpfifo:", mark)
+  print("_dump_gpfifo:", mark)
   for start, size in gpus_fifo:
     gpfifo_controls = nv_gpu.AmpereAControlGPFifo.from_address(start+size*8)
     gpfifo = to_mv(start, size * 8).cast("Q")
@@ -205,7 +293,7 @@ def make_qmd_struct_type():
     fields.append((name.replace("NVC6C0_QMDV03_00_", "").lower(), ctypes.c_uint32, data[0]-data[1]+1))
     if len(fields) >= 2 and fields[-2][0].endswith('_lower') and fields[-1][0].endswith('_upper') and fields[-1][0][:-6] == fields[-2][0][:-6]:
       fields = fields[:-2] + [(fields[-1][0][:-6], ctypes.c_uint64, fields[-1][2] + fields[-2][2])]
-  return init_c_struct_t(tuple(fields))
+  return init_c_struct_t(0x40 * 4, tuple(fields))
 qmd_struct_t = make_qmd_struct_type()
 assert ctypes.sizeof(qmd_struct_t) == 0x40 * 4
 
@@ -222,7 +310,7 @@ def _dump_qmd(address, packets):
     subc = (dat>>13) & 7
     mthd = (dat<<2) & 0x7FFF
     method_name = nvqcmds.get(mthd, f"unknown method #{mthd}")
-    if getenv("IOCTL", 0) >= 1:
+    if IOCTL_PRINT >= 1:
       print(f"\t\t{method_name}, {typ=} {size=} {subc=} {mthd=}")
       for j in range(size): print(f"\t\t\t{j}: {gpfifo[i+j+1]} | 0x{gpfifo[i+j+1]:x}")
     if mthd == 792:

extra/nv_pma/.gitignore

@@ -0,0 +1 @@
+examples/

extra/nv_pma/collect.py

@@ -0,0 +1,135 @@
+import pickle, os, sys, functools, numpy as np
+from pathlib import Path
+
+os.environ["DEV"] = "CUDA"
+os.environ["PROFILE"] = os.environ.get("PROFILE", "2")
+from extra.nv_pma.cupti import cu_prof_ext
+cu_prof_ext.enable_auto()
+
+from tinygrad import Tensor, Device
+
+if not os.environ.get("IOCTL") or not os.environ.get("GRAB_PMA"):
+  print("Usage: GRAB_PMA=1 IOCTL=1 IOCTL_PRINT=0 python3 extra/nv_pma/collect.py")
+  sys.exit(1)
+
+assert Device.DEFAULT == "CUDA", "only works with CUDA"
+
+EXAMPLES_DIR = Path(__file__).parent / "examples"
+_collectors: list[tuple[str, callable]] = []
+
+def pcsampling_test(name: str):
+  def decorator(fn):
+    @functools.wraps(fn)
+    def wrapper():
+      cu_prof_ext.clear_pma_raw_dumps()
+      cu_prof_ext.clear_cupti_pc_samples()
+
+      fn()
+      Device["CUDA"].synchronize()
+
+      dumps = cu_prof_ext.get_pma_raw_dumps()
+      # from hexdump import hexdump
+      # hexdump(dumps[0][:0x40])
+
+      return {"test_name": name, "pma_raw_dumps": list(cu_prof_ext.get_pma_raw_dumps()), "cupti_pc_samples": list(cu_prof_ext.get_cupti_pc_samples())}
+    _collectors.append((name, wrapper))
+    return wrapper
+  return decorator
+
+# Refs
+
+@pcsampling_test("test_plus")
+def test_plus():
+  a = Tensor([1, 2, 3, 4])
+  b = Tensor([5, 6, 7, 8])
+  (a + b).realize()
+
+@pcsampling_test("test_matmul")
+def test_matmul():
+  a = Tensor(np.random.rand(12, 12).astype(np.float32))
+  b = Tensor(np.random.rand(12, 12).astype(np.float32))
+  (a @ b).realize()
+
+@pcsampling_test("test_reduce_sum")
+def test_reduce_sum():
+  a = Tensor(np.random.rand(1024).astype(np.float32))
+  a.sum().realize()
+
+@pcsampling_test("test_reduce_max")
+def test_reduce_max():
+  a = Tensor(np.random.rand(1024).astype(np.float32))
+  a.max().realize()
+
+@pcsampling_test("test_exp")
+def test_exp():
+  a = Tensor(np.random.rand(256).astype(np.float32))
+  a.exp().realize()
+
+@pcsampling_test("test_softmax")
+def test_softmax():
+  a = Tensor(np.random.rand(64, 64).astype(np.float32))
+  a.softmax().realize()
+
+@pcsampling_test("test_conv2d")
+def test_conv2d():
+  x = Tensor(np.random.rand(1, 3, 32, 32).astype(np.float32))
+  w = Tensor(np.random.rand(8, 3, 3, 3).astype(np.float32))
+  x.conv2d(w).realize()
+
+@pcsampling_test("test_large_matmul")
+def test_large_matmul():
+  a = Tensor(np.random.rand(128, 128).astype(np.float32))
+  b = Tensor(np.random.rand(128, 128).astype(np.float32))
+  (a @ b).realize()
+
+@pcsampling_test("test_elementwise_chain")
+def test_elementwise_chain():
+  a = Tensor(np.random.rand(512).astype(np.float32))
+  ((a + 1) * 2 - 0.5).relu().realize()
+
+@pcsampling_test("test_broadcast")
+def test_broadcast():
+  a = Tensor(np.random.rand(64, 1).astype(np.float32))
+  b = Tensor(np.random.rand(1, 64).astype(np.float32))
+  (a + b).realize()
+
+@pcsampling_test("test_plus_big")
+def test_plus_big():
+  a = Tensor(np.random.rand(64, 32).astype(np.float32))
+  b = Tensor(np.random.rand(64, 32).astype(np.float32))
+  (a + b).realize()
+
+def save_example(name: str, data: dict):
+  pma_bytes = sum(len(d) for d in data['pma_raw_dumps'])
+  cupti_samples = sum(r['samples'] for r in data['cupti_pc_samples'])
+  print(f"  PMA: {len(data['pma_raw_dumps'])} buffers, {pma_bytes} bytes")
+  print(f"  CUPTI: {len(data['cupti_pc_samples'])} records, {cupti_samples} samples")
+
+  outfile = EXAMPLES_DIR / f"{name}.pkl"
+  with open(outfile, "wb") as f:
+    pickle.dump(data, f)
+  print(f"  Saved to {outfile}")
+
+if __name__ == "__main__":
+  EXAMPLES_DIR.mkdir(exist_ok=True)
+
+  # Run specific tests if provided as arguments, otherwise run all
+  if len(sys.argv) > 1:
+    test_names = sys.argv[1:]
+    collectors = [(name, fn) for name, fn in _collectors if name in test_names]
+    if not collectors:
+      print(f"Unknown tests: {test_names}")
+      print(f"Available: {[name for name, _ in _collectors]}")
+      sys.exit(1)
+  else:
+    collectors = _collectors
+
+  for name, collect_fn in collectors:
+    print(f"\nCollecting {name}...")
+    try:
+      data = collect_fn()
+      save_example(name, data)
+    except Exception as e:
+      print(f"  ERROR: {e}")
+      import traceback
+      traceback.print_exc()

extra/nv_pma/cupti/__init__.py

@@ -0,0 +1,26 @@
+# CUPTI autogen loader for nv_pma
+# To regenerate: REGEN=1 python -c "import extra.nv_pma.cupti"
+import importlib, pathlib
+from tinygrad.helpers import getenv
+
+root = pathlib.Path(__file__).parents[3]
+here = pathlib.Path(__file__).parent
+
+def load(name, dll, files, **kwargs):
+  if not (f:=here/f"{name}.py").exists() or getenv('REGEN'):
+    kwargs['args'] = kwargs.get('args', [])
+    f.write_text(importlib.import_module("tinygrad.runtime.support.autogen").gen(name, dll, files, **kwargs))
+  return importlib.import_module(f"extra.nv_pma.cupti.{name}")
+
+def __getattr__(nm):
+  match nm:
+    case "cupti":
+      return load("cupti", "'/usr/local/cuda/targets/x86_64-linux/lib/libcupti.so'", [
+        "/usr/local/cuda/include/cupti_result.h", "/usr/local/cuda/include/cupti_activity.h",
+        "/usr/local/cuda/include/cupti_callbacks.h", "/usr/local/cuda/include/cupti_events.h",
+        "/usr/local/cuda/include/cupti_metrics.h", "/usr/local/cuda/include/cupti_driver_cbid.h",
+        "/usr/local/cuda/include/cupti_runtime_cbid.h", "/usr/local/cuda/include/cupti_profiler_target.h",
+        "/usr/local/cuda/include/cupti_profiler_host.h", "/usr/local/cuda/include/cupti_pmsampling.h",
+        "/usr/local/cuda/include/generated_cuda_meta.h", "/usr/local/cuda/include/generated_cuda_runtime_api_meta.h"
+      ], args=["-D__CUDA_API_VERSION_INTERNAL", "-I/usr/local/cuda/include"], parse_macros=False)
+    case _: raise AttributeError(f"no such autogen: {nm}")

extra/nv_pma/cupti/cu_prof_ext.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+import ctypes
+from tinygrad.helpers import DEBUG, getenv
+from extra.nv_pma.cupti import cupti
+
+def stall_reason_name(reason: int) -> str:
+  name = cupti.CUpti_ActivityPCSamplingStallReason.get(reason)
+  return name.replace("CUPTI_ACTIVITY_PC_SAMPLING_STALL_", "").lower() if name else str(reason)
+
+class CUPTIProfiler:
+  def __init__(self):
+    self.initialized = False
+    self.pc_sampling_enabled = False
+    self.buffers: list[ctypes.Array] = []
+    self.kernel_stalls: dict[int, dict[int, int]] = {}
+    self.raw_buffers: list[bytes] = []
+    self.pc_samples: list[dict] = []
+
+  def _check_cupti(self, status, soft=False):
+    if status != cupti.CUPTI_SUCCESS:
+      if soft: return False
+      raise RuntimeError(f"CUPTI Error {status}")
+    return True
+
+  def init(self, ctx, device_id: int = 0, profile_level: int = 2):
+    if self.initialized: return
+
+    # Initialize profiler API
+    init_params = cupti.CUpti_Profiler_Initialize_Params()
+    init_params.structSize = 16
+    cupti.cuptiProfilerInitialize(ctypes.byref(init_params))
+
+    # Register buffer callbacks for Activity API
+    self._buf_req_cb = cupti.CUpti_BuffersCallbackRequestFunc(self._buffer_requested)
+    self._buf_comp_cb = cupti.CUpti_BuffersCallbackCompleteFunc(self._buffer_completed)
+    self._check_cupti(cupti.cuptiActivityRegisterCallbacks(self._buf_req_cb, self._buf_comp_cb))
+
+    # PROFILE=1: kernel timing, PROFILE=2: PC sampling with stall reasons
+    if profile_level >= 2:
+      # PC sampling for stall analysis (requires elevated privileges)
+      if DEBUG >= 1: print("  CUPTI: PC sampling mode (before)")
+      pc_status = cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING)
+      if pc_status == cupti.CUPTI_SUCCESS:
+        config = cupti.CUpti_ActivityPCSamplingConfig()
+        config.size, config.samplingPeriod = 16, cupti.CUPTI_ACTIVITY_PC_SAMPLING_PERIOD_MIN
+        cfg_status = cupti.dll.cuptiActivityConfigurePCSampling(ctx, ctypes.byref(config))
+        if cfg_status == cupti.CUPTI_SUCCESS:
+          if DEBUG >= 1: print("  CUPTI: PC sampling mode (before stall analysis)")
+          cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING_RECORD_INFO)
+          self.pc_sampling_enabled = True
+          if DEBUG >= 1: print("  CUPTI: PC sampling mode (stall analysis)")
+        elif cfg_status == 35:
+          if DEBUG >= 1: print("  CUPTI: PC sampling needs: echo 'options nvidia NVreg_RestrictProfilingToAdminUsers=0'|sudo tee /etc/modprobe.d/nvidia.conf && sudo reboot")
+      # Fall back to kernel timing if PC sampling setup failed
+      if not self.pc_sampling_enabled:
+        self._check_cupti(cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_KERNEL))
+    else:
+      # Kernel activity tracing for timing
+      self._check_cupti(cupti.cuptiActivityEnable(cupti.CUPTI_ACTIVITY_KIND_KERNEL))
+
+    self.initialized = True
+
+  def _buffer_requested(self, buffer, size, max_num_records):
+    buf = (ctypes.c_uint8 * 1024 * 1024)()  # 1MB buffer
+    self.buffers.append(buf)
+    buffer[0] = ctypes.cast(buf, ctypes.POINTER(ctypes.c_uint8))
+    size[0] = ctypes.sizeof(buf)
+    max_num_records[0] = 0
+
+  def _buffer_completed(self, ctx, stream_id, buffer, size, valid_size):
+    if valid_size > 0:
+      record = ctypes.POINTER(cupti.CUpti_Activity)()
+      while cupti.cuptiActivityGetNextRecord(buffer, valid_size, ctypes.byref(record)) == cupti.CUPTI_SUCCESS:
+        kind = record.contents.kind
+        if kind == cupti.CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL:
+          kernel = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityKernel9)).contents
+          name = ctypes.string_at(kernel.name).decode() if kernel.name else "unknown"
+          duration_us = (kernel.end - kernel.start) / 1000.0
+          grid, block = (kernel.gridX, kernel.gridY, kernel.gridZ), (kernel.blockX, kernel.blockY, kernel.blockZ)
+          print(f"  CUPTI: {name[:40]:40s} | {duration_us:10.2f} us | grid={grid} block={block} | regs={kernel.registersPerThread:3d} smem={kernel.staticSharedMemory + kernel.dynamicSharedMemory:6d}B")
+        elif kind == cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING:
+          pc = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityPCSampling3)).contents
+          cid = pc.correlationId
+          if cid not in self.kernel_stalls: self.kernel_stalls[cid] = {}
+          self.kernel_stalls[cid][pc.stallReason] = self.kernel_stalls[cid].get(pc.stallReason, 0) + pc.samples
+          self.pc_samples.append({
+            'correlationId': pc.correlationId, 'pcOffset': pc.pcOffset, 'stallReason': pc.stallReason,
+            'samples': pc.samples, 'latencySamples': pc.latencySamples, 'functionId': pc.functionId, 'sourceLocatorId': pc.sourceLocatorId
+          })
+          if DEBUG >= 3:
+            print(f"    PC {pc.pcOffset:#x} stall={stall_reason_name(pc.stallReason)} samples={pc.samples} latency={pc.latencySamples} func={pc.functionId} src={pc.sourceLocatorId}")
+        elif kind == cupti.CUPTI_ACTIVITY_KIND_PC_SAMPLING_RECORD_INFO:
+          info = ctypes.cast(record, ctypes.POINTER(cupti.CUpti_ActivityPCSamplingRecordInfo)).contents
+          cid = info.correlationId
+          if cid in self.kernel_stalls:
+            stalls = self.kernel_stalls[cid]
+            total = sum(stalls.values())
+            if total > 0:
+              top = sorted(stalls.items(), key=lambda x: -x[1])[:5]
+              stall_str = " ".join(f"{stall_reason_name(r)}:{100*c//total}%" for r,c in top if c > 0)
+              print(f"  CUPTI stalls (corr={cid}): {total} samples | {stall_str}")
+            del self.kernel_stalls[cid]
+        else: print(f"  CUPTI: Unhandled activity kind {kind}")
+
+  def flush(self):
+    if not self.initialized: return
+    self._check_cupti(cupti.cuptiActivityFlushAll(0))
+
+# Module-level profiler instance
+_profiler: CUPTIProfiler | None = None
+
+def get_profiler() -> CUPTIProfiler | None:
+  return _profiler
+
+def get_cupti_raw_buffers() -> list[bytes]:
+  return _profiler.raw_buffers if _profiler else []
+
+def clear_cupti_raw_buffers():
+  if _profiler: _profiler.raw_buffers.clear()
+
+def get_cupti_pc_samples() -> list[dict]:
+  return _profiler.pc_samples if _profiler else []
+
+def clear_cupti_pc_samples():
+  if _profiler: _profiler.pc_samples.clear()
+
+# Raw PMA buffer access (from ioctl interception)
+def get_pma_raw_dumps() -> list[bytes]:
+  try:
+    from extra.nv_gpu_driver.nv_ioctl import get_pma_raw_dumps as _get
+    return _get()
+  except ImportError: return []
+
+def clear_pma_raw_dumps():
+  try:
+    from extra.nv_gpu_driver.nv_ioctl import clear_pma_raw_dumps as _clear
+    _clear()
+  except ImportError: pass
+
+def enable(profile_level:int=2):
+  global _profiler
+  if _profiler is not None: return
+
+  _profiler = CUPTIProfiler()
+
+  # Patch CUDADevice to initialize CUPTI profiler
+  from tinygrad.runtime.ops_cuda import CUDADevice
+  _orig_init = CUDADevice.__init__
+  _orig_sync = CUDADevice.synchronize
+
+  def _patched_init(self, device: str):
+    _orig_init(self, device)
+    device_id = int(device.split(":")[1]) if ":" in device else 0
+    _profiler.init(self.context, device_id, profile_level)
+
+  def _patched_sync(self):
+    _orig_sync(self)
+    if _profiler: _profiler.flush()
+
+  CUDADevice.__init__ = _patched_init
+  CUDADevice.synchronize = _patched_sync
+
+def enable_auto():
+  if (profile_level:=getenv("PROFILE", 0)) > 0: enable(profile_level)

extra/nv_pma/decode.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import enum, collections
+from typing import Iterator
+from tinygrad.helpers import colored
+from extra.assembly.amd.sqtt import PacketType, bits
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# STALL REASONS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class StallReason(enum.IntEnum):
+  # Based on CUpti_ActivityPCSamplingStallReason
+  INVALID = 0
+  NONE = 1              # selected, selected_not_issued
+  INST_FETCH = 2        # branch_resolving, no_instructions
+  EXEC_DEPENDENCY = 3   # short_scoreboard, wait
+  MEMORY_DEPENDENCY = 4 # long_scoreboard
+  TEXTURE = 5           # tex_throttle
+  SYNC = 6              # barrier, membar
+  CONSTANT_MEMORY = 7   # imc_miss
+  PIPE_BUSY = 8         # mio_throttle, math_pipe_throttle
+  MEMORY_THROTTLE = 9   # drain, lg_throttle
+  NOT_SELECTED = 10     # not_selected
+  OTHER = 11            # misc, dispatch_stall
+  SLEEPING = 12         # sleeping
+
+STALL_KEY_MAP_AMPERE: dict[int, StallReason] = {
+  1: StallReason.MEMORY_THROTTLE, 15: StallReason.MEMORY_THROTTLE,
+  2: StallReason.CONSTANT_MEMORY,
+  3: StallReason.SYNC,
+  6: StallReason.INST_FETCH, 11: StallReason.INST_FETCH,
+  7: StallReason.EXEC_DEPENDENCY, 10: StallReason.EXEC_DEPENDENCY,
+  9: StallReason.MEMORY_DEPENDENCY,
+  12: StallReason.PIPE_BUSY,
+  17: StallReason.OTHER, 20: StallReason.OTHER,
+  18: StallReason.NONE,
+}
+
+STALL_KEY_MAP_BLACKWELL: dict[int, StallReason] = {
+  0x01: StallReason.MEMORY_THROTTLE, 0x0e: StallReason.MEMORY_THROTTLE,
+  0x02: StallReason.SYNC,
+  0x05: StallReason.INST_FETCH, 0x0a: StallReason.INST_FETCH,
+  0x06: StallReason.EXEC_DEPENDENCY, 0x09: StallReason.EXEC_DEPENDENCY,
+  0x08: StallReason.MEMORY_DEPENDENCY,
+  0x0b: StallReason.PIPE_BUSY, 0x0f: StallReason.PIPE_BUSY,
+  0x10: StallReason.OTHER, 0x13: StallReason.OTHER,
+  0x11: StallReason.NONE,
+}
+
+# Lookup table for extracting sample bytes from 32-byte packet (bytes 0-3, 8-31, skipping header at 4-7)
+LOOKUP_28B = [0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# PACKET HEADER
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMAHeader(PacketType):
+  num_bytes  = bits[4:0]    # number of sample bytes in this packet
+  tpc_id_lo  = bits[15:8]   # TPC identifier low 8 bits
+  tpc_id_hi  = bits[27:25]  # TPC identifier high 3 bits
+  dropped    = bits[28:28]  # dropped flag (resets byte accumulator)
+  @property
+  def tpc_id(self) -> int: return self.tpc_id_lo | (self.tpc_id_hi << 8)
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 8-BYTE SAMPLE FORMAT (Ampere/Ada/Hopper)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMASampleAmpere8B(PacketType):
+  pc_raw     = bits[44:0]   # raw PC value (pc_offset = pc_raw << 4)
+  stall_key  = bits[49:45]  # stall reason key
+  wave_id    = bits[55:50]  # warp/wave identifier
+  active     = bits[62:62]  # 1 if warp was executing, 0 if scheduled but not issued
+  @property
+  def pc_offset(self) -> int: return self.pc_raw << 4
+  @property
+  def stall_reason(self) -> StallReason: return STALL_KEY_MAP_AMPERE.get(self.stall_key, StallReason.OTHER)
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# 9-BYTE SAMPLE FORMAT (Blackwell+)
+# ═══════════════════════════════════════════════════════════════════════════════
+
+class PMASampleBlackwell9B(PacketType):
+  stall_key  = bits[5:0]    # stall reason key
+  pc_raw     = bits[60:8]   # raw PC value (pc_offset = pc_raw << 4)
+  wave_hi    = bits[7:6]    # wave_id high 2 bits
+  wave_lo    = bits[71:68]  # wave_id low 4 bits
+  active     = bits[67:67]  # 1 if warp was executing, 0 if scheduled but not issued
+  @property
+  def pc_offset(self) -> int: return self.pc_raw << 4
+  @property
+  def stall_reason(self) -> StallReason: return STALL_KEY_MAP_BLACKWELL.get(self.stall_key, StallReason.OTHER)
+  @property
+  def wave_id(self) -> int: return (self.wave_hi << 4) | self.wave_lo
+
+PMASample = PMASampleAmpere8B|PMASampleBlackwell9B
+
+def decode(data: bytes, sm_version: int = 0x800) -> Iterator[tuple[PMASample, int]]:
+  use_9byte = sm_version >= 0xa04
+  record_size = 9 if use_9byte else 8
+  sample_cls = PMASampleBlackwell9B if use_9byte else PMASampleAmpere8B
+
+  tpc_state: dict[int, list[int]] = collections.defaultdict(list)
+  for pkt_idx in range(len(data) // 32):
+    pkt = data[pkt_idx * 32:(pkt_idx + 1) * 32]
+    hdr = PMAHeader.from_raw(int.from_bytes(pkt[4:8], 'little'))
+
+    if hdr.dropped: tpc_state[hdr.tpc_id].clear()
+
+    for i in range(hdr.num_bytes):
+      tpc_state[hdr.tpc_id].append(pkt[LOOKUP_28B[i]])
+
+    while len(tpc_state[hdr.tpc_id]) >= record_size:
+      yield sample_cls.from_raw(int.from_bytes(bytes(tpc_state[hdr.tpc_id][:record_size]), 'little')), hdr.tpc_id
+      del tpc_state[hdr.tpc_id][:record_size]
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CLI
+# ═══════════════════════════════════════════════════════════════════════════════
+
+STALL_COLORS = {
+  StallReason.NONE: "green", StallReason.INST_FETCH: "yellow", StallReason.EXEC_DEPENDENCY: "cyan",
+  StallReason.MEMORY_DEPENDENCY: "red", StallReason.SYNC: "magenta", StallReason.CONSTANT_MEMORY: "blue",
+  StallReason.PIPE_BUSY: "yellow", StallReason.MEMORY_THROTTLE: "RED", StallReason.OTHER: "white",
+}
+
+def decode_tpc_id(tpc_id:int) -> tuple[int, int, int]:
+  # NOTE: valid only for ops_nv, cuda encoding is different
+  return (tpc_id >> 5, (tpc_id >> 1) & 0xf, tpc_id & 1)
+
+def print_samples(samples:list[tuple[PMASample, int]]) -> None:
+  if not samples: return
+  base_pc = min(s.pc_offset for s, _ in samples)
+  for s, tpc_id in samples:
+    gpc, tpc, sm = decode_tpc_id(tpc_id)
+    stall_str = colored(f"{s.stall_reason.name:17}", STALL_COLORS.get(s.stall_reason, "white"))
+    print(f"pc=0x{s.pc_offset - base_pc:06x} {stall_str} ev={s.stall_key:2d} active={s.active} wave={s.wave_id:2d} gpc={gpc} tpc={tpc} sm={sm}")
+
+def print_packets(data:bytes, sm_version:int=0x800) -> None:
+  record_size = 9 if sm_version >= 0x890 else 8
+  tpc_state: dict[int, list[int]] = collections.defaultdict(list)
+  for i in range(len(data) // 32):
+    pkt = data[i * 32:(i + 1) * 32]
+    hdr = PMAHeader.from_raw(int.from_bytes(pkt[4:8], 'little'))
+    if hdr.dropped: tpc_state[hdr.tpc_id].clear()
+    for j in range(hdr.num_bytes): tpc_state[hdr.tpc_id].append(pkt[LOOKUP_28B[j]])
+    # Show complete records extracted from this packet
+    records = []
+    while len(tpc_state[hdr.tpc_id]) >= record_size:
+      records.append(bytes(tpc_state[hdr.tpc_id][:record_size]).hex())
+      del tpc_state[hdr.tpc_id][:record_size]
+    leftover = len(tpc_state[hdr.tpc_id])
+    print(f"Pkt {i:3d}: tpc={hdr.tpc_id:4d} n={hdr.num_bytes:2d} drop={hdr.dropped} left={leftover} | {' '.join(records)}")
+
+def print_aggregated(samples:list[tuple[PMASample, int]]) -> None:
+  if not samples: return
+  base_pc = min(s.pc_offset for s, _ in samples)
+  counter: collections.Counter[tuple[int, StallReason]] = collections.Counter((s.pc_offset, s.stall_reason) for s, _ in samples)
+  print(f"\nAggregated samples (base_pc=0x{base_pc:x}):")
+  for (pc, reason), cnt in sorted(counter.items()):
+    stall_str = colored(f"{reason.name:17}", STALL_COLORS.get(reason, "white"))
+    print(f"  pc=0x{pc - base_pc:06x} {stall_str} samples={cnt:4d}")
+
+if __name__ == "__main__":
+  import sys, pickle
+
+  if len(sys.argv) < 2:
+    print("Usage: python decode.py <pkl_file> [--raw] [--sm=0xNNN]")
+    sys.exit(1)
+
+  with open(sys.argv[1], "rb") as f:
+    data = pickle.load(f)
+
+  if isinstance(data, dict):
+    sm_version = 0x800  # default to Ampere
+    for arg in sys.argv:
+      if arg.startswith("--sm="): sm_version = int(arg[5:], 0)
+    dumps = [(i, x, sm_version) for i, x in enumerate(data["pma_raw_dumps"])]
+  else:
+    devs = {e.device: e for e in data if type(e).__name__ == "ProfileDeviceEvent"}
+    dumps = []
+    for i, e in enumerate(e for e in data if type(e).__name__ == "ProfilePMAEvent"):
+      dumps.append((i, e.blob, devs[e.device].props.get('sm_version', 0x800)))
+
+  for dump_idx, raw, sm_ver in dumps:
+    print(f"\n{'='*60}\nDump {dump_idx} ({len(raw)} bytes, {len(raw)//32} packets)\n{'='*60}")
+    if "--raw" in sys.argv: print_packets(raw, sm_ver)
+    else:
+      samples = list(decode(raw, sm_ver))
+      print(f"\nDecoded {len(samples)} samples:")
+      print_samples(samples)
+      print_aggregated(samples)

extra/nv_pma/test/test_nvprof.py

@@ -0,0 +1,76 @@
+import pickle, unittest
+from collections import Counter
+from pathlib import Path
+
+from extra.nv_pma.decode import decode
+from tinygrad.helpers import DEBUG
+
+EXAMPLES_DIR = Path(__file__).parent.parent / "examples"
+EXAMPLES_5090_DIR = Path(__file__).parent.parent / "examples_5090"
+
+def decode_and_aggregate(raw_dumps: list[bytes], sm_version: int = 0x800) -> Counter[tuple[int, int]]:
+  """Decode all PMA buffers and aggregate by (relative_pc, stall_reason). Each dump is normalized separately."""
+  result: Counter[tuple[int, int]] = Counter()
+  for raw in raw_dumps:
+    samples = [s for s, _ in decode(raw, sm_version)]
+    if not samples: continue
+    base_pc = min(s.pc_offset for s in samples)
+    result += Counter((s.pc_offset - base_pc, int(s.stall_reason)) for s in samples)
+  return result
+
+def cupti_to_counter(cupti_records: list[dict]) -> Counter[tuple[int, int]]:
+  """Convert CUPTI records to Counter[(pcOffset, stallReason)]."""
+  counter: Counter[tuple[int, int]] = Counter()
+  for r in cupti_records:
+    counter[(r['pcOffset'], r['stallReason'])] += r['samples']
+  return counter
+
+class TestNVProf(unittest.TestCase):
+  def _test_example(self, name: str, sm_version: int = 0x800, examples_dir: Path = EXAMPLES_DIR):
+    pkl_file = examples_dir / f"{name}.pkl"
+    if not pkl_file.exists():
+      self.skipTest(f"Example data not found: {pkl_file}. Run collect.py first.")
+
+    with open(pkl_file, "rb") as f:
+      data = pickle.load(f)
+
+    self.assertEqual(data["test_name"], name)
+    pma_agg = decode_and_aggregate(data["pma_raw_dumps"], sm_version)
+    cupti_agg = cupti_to_counter(data["cupti_pc_samples"])
+
+    if DEBUG >= 2:
+      total = sum(cupti_agg.values())
+      mismatched = sum(abs(pma_agg.get(k, 0) - v) for k, v in cupti_agg.items())
+      mismatched += sum(v for k, v in pma_agg.items() if k not in cupti_agg)
+      mismatched //= 2
+
+      print(f"\n=== Test: {name} ===")
+      print(f"Total samples: {total}, Mismatched: {mismatched} ({mismatched/total*100 if total else 0:.1f}%)")
+
+    self.assertEqual(pma_agg, cupti_agg, f"PMA: {dict(pma_agg)}\nCUPTI: {dict(cupti_agg)}")
+
+  # Ampere tests (8-byte format)
+  def test_decode_test_plus(self): self._test_example("test_plus")
+  def test_decode_test_reduce_sum(self): self._test_example("test_reduce_sum")
+  def test_decode_test_broadcast(self): self._test_example("test_broadcast")
+  def test_decode_test_matmul(self): self._test_example("test_matmul")
+  def test_decode_test_plus_big(self): self._test_example("test_plus_big")
+  def test_decode_test_elementwise_chain(self): self._test_example("test_elementwise_chain")
+  def test_decode_test_conv2d(self): self._test_example("test_conv2d")
+  def test_decode_test_large_matmul(self): self._test_example("test_large_matmul")
+
+  # Blackwell/5090 tests (9-byte format)
+  def test_5090_test_plus(self): self._test_example("test_plus", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_plus_big(self): self._test_example("test_plus_big", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_broadcast(self): self._test_example("test_broadcast", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_matmul(self): self._test_example("test_matmul", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_large_matmul(self): self._test_example("test_large_matmul", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_reduce_sum(self): self._test_example("test_reduce_sum", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_reduce_max(self): self._test_example("test_reduce_max", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_elementwise_chain(self): self._test_example("test_elementwise_chain", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_conv2d(self): self._test_example("test_conv2d", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_exp(self): self._test_example("test_exp", 0xa04, EXAMPLES_5090_DIR)
+  def test_5090_test_softmax(self): self._test_example("test_softmax", 0xa04, EXAMPLES_5090_DIR)
+
+if __name__ == "__main__":
+  unittest.main()

extra/optimization/generate_dataset.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
+set -e
 export PAGE_SIZE=1
 export PYTHONPATH=.
 export LOGOPS=/tmp/ops
 export CAPTURE_PROCESS_REPLAY=1
-rm $LOGOPS
+rm "$LOGOPS" 2>/dev/null || true
 test/external/process_replay/reset.py
 
-CI=1 python3 -m pytest -n=auto test/test_ops.py test/test_nn.py test/test_winograd.py test/models/test_real_world.py --durations=20
+CI=1 python3 -m pytest -n=auto test/test_ops.py test/test_nn.py test/unit/test_winograd.py test/models/test_real_world.py --durations=20
 CL=1 python3 -m pytest test/test_tiny.py
 
 # extract, sort and uniq

extra/qcom_gpu_driver/opencl_ioctl.py

@@ -42,7 +42,7 @@ def get_struct(argp, stype):
 
 def format_struct(s):
   sdats = []
-  for field_name, field_type in s._fields_:
+  for field_name, *_ in s._real_fields_:
     if field_name in {"__pad", "PADDING_0"}: continue
     dat = getattr(s, field_name)
     if isinstance(dat, int): sdats.append(f"{field_name}:0x{dat:X}")

extra/sqtt/active_sqtt_parse.py

@@ -11,7 +11,7 @@ from tinygrad import Tensor
 from tinygrad.helpers import system, OSX
 from tinygrad.runtime.ops_amd import AMDProgram
 from extra.sqtt.roc import decode, WaveExec, ProfileSQTTEvent
-from tinygrad.device import Device, ProfileDeviceEvent
+from tinygrad.device import Device
 
 from extra.sqtt.attempt_sqtt_parse import parse_sqtt_print_packets
 
@@ -23,7 +23,7 @@ def save_sqtt():
   dev.profile_events.clear()
   sqtt:dict[str, list[WaveExec]] = {}
   yield sqtt
-  events = dev.profile_events+[ProfileDeviceEvent("AMD", props=dev.device_props())]
+  events = dev.profile_events
 
   #rctx = decode(events)
   #assert len(rctx.inst_execs) > 0, "empty sqtt output"
@@ -45,6 +45,7 @@ matmul:
 .rodata
 .p2align 6
 .amdhsa_kernel matmul
+  .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_next_free_vgpr .amdgcn.next_free_vgpr
   .amdhsa_next_free_sgpr .amdgcn.next_free_sgpr

extra/sqtt/examples/generate_examples.py

@@ -0,0 +1,23 @@
+import os, subprocess
+from pathlib import Path
+from tinygrad.helpers import temp
+
+EXAMPLES_DIR = Path(__file__).parent
+PROFILE_PATH = Path(temp("profile.pkl", append_user=True))
+
+EXAMPLES = [
+  "test.test_custom_kernel.TestCustomKernel.test_empty",
+  "test.test_tiny.TestTiny.test_plus",
+  "test.test_tiny.TestTiny.test_gemm",
+]
+
+if __name__ == "__main__":
+  arch = subprocess.check_output(["python", "-c", "from tinygrad import Device; print(Device['AMD'].arch)"], text=True,
+                                 env={**os.environ, "DEBUG":"0"}).rstrip()
+  (EXAMPLES_DIR/arch).mkdir(exist_ok=True)
+  for test in EXAMPLES:
+    for i in range(2):
+      subprocess.run(["python", "-m", "unittest", test], cwd=EXAMPLES_DIR.parent.parent.parent,
+                     env={**os.environ, "AMD":"1", "SQTT_LIMIT_SE":"-1", "VIZ":"-2"}, check=True)
+      PROFILE_PATH.rename(dest:=EXAMPLES_DIR/arch/f"profile_{test.split('.')[-1].replace('test_', '')}_run_{i}.pkl")
+      print(f"saved SQTT trace to {dest}")