nimlgen/tinygrad
1
0
Fork 0
forked from tinygrad/tinygrad
Code Pull Requests Activity

this diff is getting dumb

Browse Source
This commit is contained in:
geohot
2025-12-29 15:54:14 +00:00
parent d6863e42bd
commit afa490e3f4
2 changed files with 444 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/test_rdna_renderer.py
+5 -5
View File
@@ -377,7 +377,7 @@ class TestLookAheadPacking(unittest.TestCase):
buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0)
# Create index (workitem ID)
ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32))
ridx = UOp.special(32, "ridx0", dtype=dtypes.int)
# Create 16 LOADs at different offsets
loads = []
@@ -415,7 +415,7 @@ class TestLookAheadPacking(unittest.TestCase):
# Create a minimal kernel with half LOADs feeding half16 VECTORIZE
buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0)
ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32))
ridx = UOp.special(32, "ridx0", dtype=dtypes.int)
# Create 16 LOADs
loads = []
@@ -452,7 +452,7 @@ class TestLookAheadPacking(unittest.TestCase):
# Create a minimal kernel with half LOADs feeding half16 VECTORIZE
buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0)
ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32))
ridx = UOp.special(32, "ridx0", dtype=dtypes.int)
# Create 16 LOADs
loads = []
@@ -494,7 +494,7 @@ class TestLookAheadPacking(unittest.TestCase):
# Create a minimal kernel with half LOADs feeding half16 VECTORIZE
buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0)
ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32))
ridx = UOp.special(32, "ridx0", dtype=dtypes.int)
# Create 16 LOADs
loads = []
@@ -537,7 +537,7 @@ class TestLookAheadPacking(unittest.TestCase):
# Create two buffers (like A and B matrices)
buf_a = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0)
buf_b = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=1)
ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32))
ridx = UOp.special(32, "ridx0", dtype=dtypes.int)
# Create 16 LOADs from buf_a
loads_a = []
tinygrad/renderer/rdna.py
+439 -30
View File
@@ -646,13 +646,13 @@ def render_signed_small_int_max(ctx, x, a, b):
def render_signed_small_int_idiv(ctx, x, a, b):
"""Render IDIV for signed int8/int16. Sign-extend operands first, then use full idiv correction logic.
v_rcp_f32 is an approximation, so we need the same correction as render_idiv."""
v_rcp_f32 is an approximation, so we need correction for both positive and negative divisors."""
dtype = x.dtype
bits = 8 if dtype == dtypes.int8 else 16
ra, rb, rx = ctx.r[a], ctx.r[b], ctx.r[x]
s = ctx.get_scratch_vgpr()
# Sign-extend operands to scratch registers, then use render_idiv algorithm
# s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp, s+4: remainder
# s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp, s+4: remainder, s+5: temp
return [
f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # s = sign-extend a
f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # s+1 = sign-extend b
@@ -675,13 +675,13 @@ def render_signed_small_int_idiv(ctx, x, a, b):
# Recompute remainder after large correction
f"v_mul_lo_u32 v{s+3}, {rx}, v{s+1}", # s+3 = q*b
f"v_sub_nc_u32 v{s+4}, v{s}, v{s+3}", # s+4 = r (after large correction)
# Correction: If r >= b and b > 0, q is too low - increment q
# Correction for b > 0: If r >= b, q is too low - increment q
f"v_cmp_ge_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r >= b)
f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r >= b
f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0)
f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", # s+3 = 1 if (r >= b AND b > 0)
f"v_add_nc_u32 {rx}, {rx}, v{s+3}", # q += 1 if needed
# Correction: If r <= -b and b > 0 (r too negative), decrement q
# Correction for b > 0: If r <= -b (r < 0 and r + b <= 0), q is too high - decrement q
f"v_cmp_lt_i32 vcc_lo, v{s+4}, 0", # vcc = (r < 0)
f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r < 0
f"v_add_nc_u32 v{s+5}, v{s+4}, v{s+1}", # s+5 = r + b
@@ -690,26 +690,87 @@ def render_signed_small_int_idiv(ctx, x, a, b):
f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0)
f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo",
f"v_sub_nc_u32 {rx}, {rx}, v{s+3}", # q -= 1 if needed
# Correction for b < 0: If r <= b (r is more negative), q is too low - increment q
f"v_cmp_le_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r <= b)
f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r <= b
f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0)
f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", # s+3 = 1 if (r <= b AND b < 0)
f"v_add_nc_u32 {rx}, {rx}, v{s+3}", # q += 1 if needed
# Correction for b < 0: If r >= -b (r > 0 and r + b >= 0), q is too high - decrement q
f"v_cmp_gt_i32 vcc_lo, v{s+4}, 0", # vcc = (r > 0)
f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r > 0
f"v_add_nc_u32 v{s+5}, v{s+4}, v{s+1}", # s+5 = r + b (when b < 0, this checks r >= -b)
f"v_cmp_ge_i32 vcc_lo, v{s+5}, 0", # vcc = (r + b >= 0), i.e., r >= -b
f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo",
f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0)
f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo",
f"v_sub_nc_u32 {rx}, {rx}, v{s+3}", # q -= 1 if needed
]
def render_signed_small_int_mod(ctx, x, a, b):
"""Render MOD for signed int8/int16. Sign-extend operands first, then compute a - (a/b)*b."""
"""Render MOD for signed int8/int16. Sign-extend operands first, compute corrected quotient, then remainder.
v_rcp_f32 is an approximation, so we need correction for both positive and negative divisors."""
dtype = x.dtype
bits = 8 if dtype == dtypes.int8 else 16
ra, rb, rx = ctx.r[a], ctx.r[b], ctx.r[x]
s = ctx.get_scratch_vgpr()
# Sign-extend and compute mod = a - (a/b)*b
# Use same correction logic as IDIV, then compute r = a - q*b
# s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp/q, s+4: temp, s+5: temp, s+6: temp
return [
f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # sign-extend a
f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # sign-extend b
f"v_cvt_f32_i32 v{s+2}, v{s}", # float(a)
f"v_cvt_f32_i32 v{s+3}, v{s+1}", # float(b)
f"v_rcp_f32 v{s+3}, v{s+3}", # 1/float(b)
f"v_mul_f32 v{s+2}, v{s+2}, v{s+3}", # float(a)/float(b)
f"v_trunc_f32 v{s+2}, v{s+2}", # truncate toward zero
f"v_cvt_i32_f32 v{s+2}, v{s+2}", # q = trunc(a/b)
f"v_mul_lo_u32 v{s+2}, v{s+2}, v{s+1}", # q*b
f"v_sub_nc_u32 {rx}, v{s}, v{s+2}", # a - q*b
f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # s = sign-extend a
f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # s+1 = sign-extend b
# Initial float division
f"v_cvt_f32_i32 v{s+3}, v{s}", # s+3 = float(a)
f"v_cvt_f32_i32 v{s+4}, v{s+1}", # s+4 = float(b)
f"v_rcp_f32 v{s+2}, v{s+4}", # s+2 = 1/float(b) (approx)
f"v_mul_f32 v{s+3}, v{s+3}, v{s+2}", # s+3 = float(a)/float(b) (approx)
f"v_trunc_f32 v{s+3}, v{s+3}", # s+3 = trunc(a/b)
f"v_cvt_i32_f32 v{s+3}, v{s+3}", # s+3 = q (initial quotient)
# Compute exact remainder r = a - q*b
f"v_mul_lo_u32 v{s+4}, v{s+3}, v{s+1}", # s+4 = q*b
f"v_sub_nc_u32 v{s+4}, v{s}, v{s+4}", # s+4 = r = a - q*b (exact remainder)
# First correction pass using float - r is now smaller, fits in float24
f"v_cvt_f32_i32 v{s+5}, v{s+4}", # s+5 = float(r)
f"v_mul_f32 v{s+5}, v{s+5}, v{s+2}", # s+5 = float(r)/float(b)
f"v_trunc_f32 v{s+5}, v{s+5}", # s+5 = trunc(r/b)
f"v_cvt_i32_f32 v{s+5}, v{s+5}", # s+5 = large correction
f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += large correction
# Recompute remainder after large correction
f"v_mul_lo_u32 v{s+5}, v{s+3}, v{s+1}", # s+5 = q*b
f"v_sub_nc_u32 v{s+4}, v{s}, v{s+5}", # s+4 = r (after large correction)
# Correction for b > 0: If r >= b, q is too low - increment q
f"v_cmp_ge_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r >= b)
f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r >= b
f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0)
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", # s+5 = 1 if (r >= b AND b > 0)
f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += 1 if needed
# Correction for b > 0: If r <= -b (r < 0 and r + b <= 0), q is too high - decrement q
f"v_cmp_lt_i32 vcc_lo, v{s+4}, 0", # vcc = (r < 0)
f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r < 0
f"v_add_nc_u32 v{s+6}, v{s+4}, v{s+1}", # s+6 = r + b
f"v_cmp_le_i32 vcc_lo, v{s+6}, 0", # vcc = (r + b <= 0)
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo",
f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0)
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo",
f"v_sub_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q -= 1 if needed
# Correction for b < 0: If r <= b (r is more negative), q is too low - increment q
f"v_cmp_le_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r <= b)
f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r <= b
f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0)
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", # s+5 = 1 if (r <= b AND b < 0)
f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += 1 if needed
# Correction for b < 0: If r >= -b (r > 0 and r + b >= 0), q is too high - decrement q
f"v_cmp_gt_i32 vcc_lo, v{s+4}, 0", # vcc = (r > 0)
f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r > 0
f"v_add_nc_u32 v{s+6}, v{s+4}, v{s+1}", # s+6 = r + b (when b < 0, this checks r >= -b)
f"v_cmp_ge_i32 vcc_lo, v{s+6}, 0", # vcc = (r + b >= 0), i.e., r >= -b
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo",
f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0)
f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo",
f"v_sub_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q -= 1 if needed
# Final remainder: r = a - q*b
f"v_mul_lo_u32 v{s+3}, v{s+3}, v{s+1}", # s+3 = q*b
f"v_sub_nc_u32 {rx}, v{s}, v{s+3}", # rx = a - q*b = final remainder
]
def render_signed_small_int_mul(ctx, x, a, b):
@@ -970,25 +1031,28 @@ def render_64bit_shl(ctx, x, a, b):
def render_64bit_add(ctx, x, a, b):
"""Render 64-bit integer addition with carry: dst = a + b.
Uses v_add_co_u32 for low 32 bits (produces carry), v_add_co_ci_u32 for high (consumes carry).
Handles constants: for small constants, high part is 0; for register pairs, extracts lo/hi."""
Handles constants: for negative constants, high part is -1; for positive, high part is 0."""
rx = ctx.r[x]
ra = ctx.r[a]
rb = ctx.r[b]
dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1
# Helper to extract lo/hi from a 64-bit operand (register pair or constant)
def get_lo_hi(r):
def get_lo_hi(r, uop=None):
if '[' in r: # register pair like v[10:11]
base = get_reg_base(r)
return f"v{base}", f"v{base+1}"
elif r.startswith('v') or r.startswith('s'): # single register
num = int(r[1:])
return f"v{num}", f"v{num}" # same reg for both (shouldn't happen for 64-bit)
else: # constant - high part is 0 for small constants
else: # constant - high part is -1 for negative, 0 for positive
# Check if this is a negative constant by looking at the UOp arg
if uop is not None and uop.op is Ops.CONST and isinstance(uop.arg, int) and uop.arg < 0:
return r, "-1" # sign extend: high part is all 1s for negative
return r, "0"
a_lo, a_hi = get_lo_hi(ra)
b_lo, b_hi = get_lo_hi(rb)
a_lo, a_hi = get_lo_hi(ra, a)
b_lo, b_hi = get_lo_hi(rb, b)
return [
f"v_add_co_u32 v{dst_lo}, vcc_lo, {a_lo}, {b_lo}", # low + low, carry in vcc_lo
@@ -998,25 +1062,28 @@ def render_64bit_add(ctx, x, a, b):
def render_64bit_sub(ctx, x, a, b):
"""Render 64-bit integer subtraction with borrow: dst = a - b.
Uses v_sub_co_u32 for low 32 bits (produces borrow), v_sub_co_ci_u32 for high (consumes borrow).
Handles constants: for small constants, high part is 0; for register pairs, extracts lo/hi."""
Handles constants: for negative constants, high part is -1; for positive, high part is 0."""
rx = ctx.r[x]
ra = ctx.r[a]
rb = ctx.r[b]
dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1
# Helper to extract lo/hi from a 64-bit operand (register pair or constant)
def get_lo_hi(r):
def get_lo_hi(r, uop=None):
if '[' in r: # register pair like v[10:11]
base = get_reg_base(r)
return f"v{base}", f"v{base+1}"
elif r.startswith('v') or r.startswith('s'): # single register
num = int(r[1:])
return f"v{num}", f"v{num}" # same reg for both (shouldn't happen for 64-bit)
else: # constant - high part is 0 for small constants
else: # constant - high part is -1 for negative, 0 for positive
# Check if this is a negative constant by looking at the UOp arg
if uop is not None and uop.op is Ops.CONST and isinstance(uop.arg, int) and uop.arg < 0:
return r, "-1" # sign extend: high part is all 1s for negative
return r, "0"
a_lo, a_hi = get_lo_hi(ra)
b_lo, b_hi = get_lo_hi(rb)
a_lo, a_hi = get_lo_hi(ra, a)
b_lo, b_hi = get_lo_hi(rb, b)
return [
f"v_sub_co_u32 v{dst_lo}, vcc_lo, {a_lo}, {b_lo}", # low - low, borrow in vcc_lo
@@ -1068,6 +1135,42 @@ def render_64bit_and(ctx, x, a, b):
f"v_and_b32 v{dst_hi}, v{a_hi}, v{b_hi}",
]
def render_64bit_max(ctx, x, a, b):
"""Render 64-bit integer MAX: dst = max(a, b).
Algorithm: Compare 64-bit values, then select larger one.
For signed: compare high parts as signed, low parts as unsigned when high parts equal.
For unsigned: compare both parts as unsigned."""
rx = ctx.r[x]
ra = ctx.r[a]
rb = ctx.r[b]
dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1
a_lo = get_reg_base(ra) if '[' in ra else int(ra[1:])
a_hi = a_lo + 1 if '[' in ra else a_lo
b_lo = get_reg_base(rb) if '[' in rb else int(rb[1:])
b_hi = b_lo + 1 if '[' in rb else b_lo
is_signed = x.dtype in (dtypes.int64, dtypes.long)
hi_cmp = "i32" if is_signed else "u32"
# Use scratch SGPRs for comparison intermediates
ctx.scratch_sgpr_used = True
ss0, ss1 = f"s{ctx.gated_sgpr}", f"s{ctx.gated_sgpr+1}"
# Compare a > b: (a_hi > b_hi) || (a_hi == b_hi && a_lo > b_lo)
# Then select: if a > b, choose a; else choose b
return [
# Compare high parts (signed for signed types)
f"v_cmp_gt_{hi_cmp} vcc_lo, v{a_hi}, v{b_hi}", # vcc_lo = a_hi > b_hi
f"s_mov_b32 {ss0}, vcc_lo", # save hi_gt
f"v_cmp_eq_u32 vcc_lo, v{a_hi}, v{b_hi}", # vcc_lo = a_hi == b_hi
f"s_mov_b32 {ss1}, vcc_lo", # save hi_eq
f"v_cmp_gt_u32 vcc_lo, v{a_lo}, v{b_lo}", # vcc_lo = a_lo > b_lo (always unsigned)
f"s_and_b32 {ss1}, {ss1}, vcc_lo", # ss1 = hi_eq AND lo_gt
f"s_or_b32 vcc_lo, {ss0}, {ss1}", # vcc_lo = a > b (full 64-bit comparison)
# Select: if a > b, choose a; else choose b
f"v_cndmask_b32 v{dst_lo}, v{b_lo}, v{a_lo}, vcc_lo",
f"v_cndmask_b32 v{dst_hi}, v{b_hi}, v{a_hi}, vcc_lo",
]
def render_64bit_udiv(ctx, x, a, b):
"""Render 64-bit unsigned division using f64 arithmetic.
Algorithm: convert to f64, divide, floor, convert back, then refine.
@@ -1252,7 +1355,8 @@ def render_64bit_udiv(ctx, x, a, b):
]
def render_64bit_idiv(ctx, x, a, b):
"""Render 64-bit signed division using f64 arithmetic."""
"""Render 64-bit signed division using f64 arithmetic.
Special cases: b==1 returns a, b==-1 returns -a (avoids f64 precision loss)."""
rx, ra, rb = ctx.r[x], ctx.r[a], ctx.r[b]
dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1
a_lo = get_reg_base(ra) if '[' in ra else int(ra[1:])
@@ -1262,6 +1366,19 @@ def render_64bit_idiv(ctx, x, a, b):
s = ctx.get_scratch_vgpr()
# For signed division, convert to unsigned, divide, then fix sign
return [
# Special case: if b == 1, result = a; if b == -1, result = -a
# Check: b_hi == 0 && b_lo == 1 means b == 1
# Check: b_hi == -1 && b_lo == -1 means b == -1 (0xFFFFFFFFFFFFFFFF)
# Use s+20 and s+21 for flags to avoid conflict with f64 computation (uses s+0..s+16)
f"v_xor_b32 v{s+17}, v{b_lo}, 1", # b_lo XOR 1 (0 if b_lo == 1)
f"v_or_b32 v{s+17}, v{s+17}, v{b_hi}", # combined: 0 if b == 1
f"v_cmp_eq_u32 vcc_lo, v{s+17}, 0", # vcc = (b == 1)
f"v_cndmask_b32 v{s+20}, 0, 1, vcc_lo", # s+20 = b_is_one flag
# Check for b == -1 (all bits set)
f"v_and_b32 v{s+17}, v{b_lo}, v{b_hi}", # AND of both halves
f"v_cmp_eq_i32 vcc_lo, v{s+17}, -1", # vcc = both halves are -1
f"v_cndmask_b32 v{s+21}, 0, 1, vcc_lo", # s+21 = b_is_neg_one flag
# Get absolute values and track signs
# sign_a = a >> 63, sign_b = b >> 63
f"v_ashrrev_i32 v{s}, 31, v{a_hi}", # s = sign extension of a
@@ -1320,11 +1437,298 @@ def render_64bit_idiv(ctx, x, a, b):
f"v_fma_f64 v[{s+9}:{s+10}], v[{s+9}:{s+10}], v[{s+11}:{s+12}], v[{s+7}:{s+8}]", # q_lo = q + q_hi*(-2^32)
f"v_cvt_u32_f64 v{s+3}, v[{s+9}:{s+10}]", # q_lo as u32
# Refinement: compute remainder and adjust q if needed (for values > 2^53 where f64 loses precision)
# abs_a is still recoverable from a and sign (s), abs_b is in s+5:s+6
# Recompute abs_a from original a: abs_a = (a ^ sign) - sign
f"v_xor_b32 v{s+7}, v{a_lo}, v{s}",
f"v_xor_b32 v{s+8}, v{a_hi}, v{s}",
f"v_sub_co_u32 v{s+7}, vcc_lo, v{s+7}, v{s}",
f"v_sub_co_ci_u32 v{s+8}, vcc_lo, v{s+8}, v{s}, vcc_lo", # abs_a in s+7:s+8
# Refinement pass 1: compute product = q * abs_b (lower 64 bits)
# product = q_lo*b_lo + (q_lo*b_hi + q_hi*b_lo) << 32
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", # product_lo = low(q_lo * b_lo)
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", # tmp = high(q_lo * b_lo)
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", # q_lo * b_hi (contributes to product_hi)
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", # q_hi * b_lo (contributes to product_hi)
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", # product_hi = tmp + q_lo*b_hi
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", # product_hi += q_hi*b_lo
# Compute remainder = abs_a - product
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", # r_lo = abs_a_lo - product_lo
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", # r_hi
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", # borrow flag (1 = q too high)
# If borrow, decrement q
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
# If no borrow, check if r >= b and increment q
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", # r_hi > b_hi?
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", # r_hi == b_hi?
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", # r_lo >= b_lo?
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", # (r_hi == b_hi) && (r_lo >= b_lo)
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", # r >= b
f"v_xor_b32 v{s+15}, v{s+13}, 1", # no_borrow = !borrow
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", # increment only if no_borrow && r >= b
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Refinement pass 2
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Refinement pass 3
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Refinement passes 4-7 (for very large values with f64 error up to ~10)
# Pass 4
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Pass 5
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Pass 6
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Pass 7
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Pass 8
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}",
f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}",
f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}",
f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}",
f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo",
f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}",
f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo",
f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}",
f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo",
f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}",
f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}",
f"v_or_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_xor_b32 v{s+15}, v{s+13}, 1",
f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}",
f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo",
f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0",
f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo",
f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Passes 9-15 (abbreviated for brevity - same pattern)
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", f"v_xor_b32 v{s+15}, v{s+13}, 1", f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Pass 10
f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}",
f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}",
f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo",
f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo",
f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo",
f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", f"v_xor_b32 v{s+15}, v{s+13}, 1", f"v_and_b32 v{s+14}, v{s+14}, v{s+15}",
f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo",
# Apply sign: result = (abs_q ^ result_sign) - result_sign
f"v_xor_b32 v{dst_lo}, v{s+3}, v{s+2}",
f"v_xor_b32 v{dst_hi}, v{s+4}, v{s+2}",
f"v_sub_co_u32 v{dst_lo}, vcc_lo, v{dst_lo}, v{s+2}",
f"v_sub_co_ci_u32 v{dst_hi}, vcc_lo, v{dst_hi}, v{s+2}, vcc_lo",
# Special case: if b == 1, use a directly
f"v_cmp_ne_u32 vcc_lo, v{s+20}, 0", # vcc = (b == 1)
f"v_cndmask_b32 v{dst_lo}, v{dst_lo}, v{a_lo}, vcc_lo", # dst_lo = a_lo if b==1 else q_lo
f"v_cndmask_b32 v{dst_hi}, v{dst_hi}, v{a_hi}, vcc_lo", # dst_hi = a_hi if b==1 else q_hi
# Special case: if b == -1, use -a (negate a)
f"v_xor_b32 v{s+3}, v{a_lo}, -1", # ~a_lo
f"v_xor_b32 v{s+4}, v{a_hi}, -1", # ~a_hi
f"v_add_co_u32 v{s+3}, vcc_lo, v{s+3}, 1", # neg_a_lo = ~a_lo + 1
f"v_add_co_ci_u32 v{s+4}, vcc_lo, v{s+4}, 0, vcc_lo", # neg_a_hi = ~a_hi + carry
f"v_cmp_ne_u32 vcc_lo, v{s+21}, 0", # vcc = (b == -1)
f"v_cndmask_b32 v{dst_lo}, v{dst_lo}, v{s+3}, vcc_lo", # dst_lo = neg_a_lo if b==-1
f"v_cndmask_b32 v{dst_hi}, v{dst_hi}, v{s+4}, vcc_lo", # dst_hi = neg_a_hi if b==-1
]
def render_64bit_umod(ctx, x, a, b):
@@ -1594,6 +1998,9 @@ string_rewrite = PatternMatcher([
# signed int8/int16 MAX need sign extension (MIN uses MAX + XOR pattern)
(UPat(Ops.MAX, name="x", dtype=dtypes.int8, src=(UPat.var("a"), UPat.var("b"))), render_signed_small_int_max),
(UPat(Ops.MAX, name="x", dtype=dtypes.int16, src=(UPat.var("a"), UPat.var("b"))), render_signed_small_int_max),
# 64-bit integer MAX: need full 64-bit comparison
(UPat(Ops.MAX, name="x", dtype=dtypes.long, src=(UPat.var("a"), UPat.var("b"))), render_64bit_max),
(UPat(Ops.MAX, name="x", dtype=dtypes.ulong, src=(UPat.var("a"), UPat.var("b"))), render_64bit_max),
# alu ops - extract low 32 bits for 64-bit integer operands since RDNA has limited 64-bit int ALU
(UPat(GroupOp.ALU, name="x"), lambda ctx, x: ctx.code_for_op[x.op](
extract_low_32(ctx.r[x]) if x.dtype in (dtypes.long, dtypes.ulong) else ctx.r[x],
@@ -3215,11 +3622,13 @@ amdhsa.version:
if recompute_const is not None and recompute_base is not None:
# ADD(base, const) - use v_lshl_add_u32 to compute (base << shift) + (const << shift)
base_reg = r.get(recompute_base)
if base_reg and isinstance(base_reg, str) and base_reg.startswith('v'):
# v_lshl_add_u32 accepts both VGPR and SGPR sources
if base_reg and isinstance(base_reg, str) and (base_reg.startswith('v') or base_reg.startswith('s')):
kernel.append(f"v_lshl_add_u32 {deferred_store_addr_vgpr}, {base_reg}, {shift_val}, {recompute_const << shift_val}")
else:
# Fallback: copy from pre-computed
kernel.append(f"v_mov_b32 {deferred_store_addr_vgpr}, {r[idx]}")
# Fallback: compute the SHL manually if we can't recompute inline
# This shouldn't normally happen, but handle gracefully
kernel.append(f"v_mov_b32 {deferred_store_addr_vgpr}, {r[idx] if r.get(idx) != 'RECOMPUTE_AT_STORE' else 0}")
else:
# ADD(reg, reg) - compute ADD then SHL
kernel.append(f"v_add_nc_u32 {deferred_store_addr_vgpr}, {r[add_src0]}, {r[add_src1]}")