From afa490e3f43861b29bd3172d3477b3f844b4d2fd Mon Sep 17 00:00:00 2001 From: George Hotz Date: Mon, 22 Dec 2025 21:44:18 +0000 Subject: [PATCH] this diff is getting dumb --- test/test_rdna_renderer.py | 10 +- tinygrad/renderer/rdna.py | 469 ++++++++++++++++++++++++++++++++++--- 2 files changed, 444 insertions(+), 35 deletions(-) diff --git a/test/test_rdna_renderer.py b/test/test_rdna_renderer.py index 3a67606a1c..3843c544ad 100644 --- a/test/test_rdna_renderer.py +++ b/test/test_rdna_renderer.py @@ -377,7 +377,7 @@ class TestLookAheadPacking(unittest.TestCase): buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0) # Create index (workitem ID) - ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32)) + ridx = UOp.special(32, "ridx0", dtype=dtypes.int) # Create 16 LOADs at different offsets loads = [] @@ -415,7 +415,7 @@ class TestLookAheadPacking(unittest.TestCase): # Create a minimal kernel with half LOADs feeding half16 VECTORIZE buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0) - ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32)) + ridx = UOp.special(32, "ridx0", dtype=dtypes.int) # Create 16 LOADs loads = [] @@ -452,7 +452,7 @@ class TestLookAheadPacking(unittest.TestCase): # Create a minimal kernel with half LOADs feeding half16 VECTORIZE buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0) - ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32)) + ridx = UOp.special(32, "ridx0", dtype=dtypes.int) # Create 16 LOADs loads = [] @@ -494,7 +494,7 @@ class TestLookAheadPacking(unittest.TestCase): # Create a minimal kernel with half LOADs feeding half16 VECTORIZE buf = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0) - ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32)) + ridx = UOp.special(32, "ridx0", dtype=dtypes.int) # Create 16 LOADs loads = [] @@ -537,7 +537,7 @@ class TestLookAheadPacking(unittest.TestCase): # Create two buffers (like A and B matrices) buf_a = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=0) buf_b = UOp(Ops.DEFINE_GLOBAL, dtypes.half.ptr(), arg=1) - ridx = UOp(Ops.SPECIAL, dtypes.int, (), ("ridx0", 32)) + ridx = UOp.special(32, "ridx0", dtype=dtypes.int) # Create 16 LOADs from buf_a loads_a = [] diff --git a/tinygrad/renderer/rdna.py b/tinygrad/renderer/rdna.py index 39c0d99701..a16e801aa1 100644 --- a/tinygrad/renderer/rdna.py +++ b/tinygrad/renderer/rdna.py @@ -646,13 +646,13 @@ def render_signed_small_int_max(ctx, x, a, b): def render_signed_small_int_idiv(ctx, x, a, b): """Render IDIV for signed int8/int16. Sign-extend operands first, then use full idiv correction logic. - v_rcp_f32 is an approximation, so we need the same correction as render_idiv.""" + v_rcp_f32 is an approximation, so we need correction for both positive and negative divisors.""" dtype = x.dtype bits = 8 if dtype == dtypes.int8 else 16 ra, rb, rx = ctx.r[a], ctx.r[b], ctx.r[x] s = ctx.get_scratch_vgpr() # Sign-extend operands to scratch registers, then use render_idiv algorithm - # s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp, s+4: remainder + # s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp, s+4: remainder, s+5: temp return [ f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # s = sign-extend a f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # s+1 = sign-extend b @@ -675,13 +675,13 @@ def render_signed_small_int_idiv(ctx, x, a, b): # Recompute remainder after large correction f"v_mul_lo_u32 v{s+3}, {rx}, v{s+1}", # s+3 = q*b f"v_sub_nc_u32 v{s+4}, v{s}, v{s+3}", # s+4 = r (after large correction) - # Correction: If r >= b and b > 0, q is too low - increment q + # Correction for b > 0: If r >= b, q is too low - increment q f"v_cmp_ge_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r >= b) f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r >= b f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0) f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", # s+3 = 1 if (r >= b AND b > 0) f"v_add_nc_u32 {rx}, {rx}, v{s+3}", # q += 1 if needed - # Correction: If r <= -b and b > 0 (r too negative), decrement q + # Correction for b > 0: If r <= -b (r < 0 and r + b <= 0), q is too high - decrement q f"v_cmp_lt_i32 vcc_lo, v{s+4}, 0", # vcc = (r < 0) f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r < 0 f"v_add_nc_u32 v{s+5}, v{s+4}, v{s+1}", # s+5 = r + b @@ -690,26 +690,87 @@ def render_signed_small_int_idiv(ctx, x, a, b): f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0) f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", f"v_sub_nc_u32 {rx}, {rx}, v{s+3}", # q -= 1 if needed + # Correction for b < 0: If r <= b (r is more negative), q is too low - increment q + f"v_cmp_le_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r <= b) + f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r <= b + f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0) + f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", # s+3 = 1 if (r <= b AND b < 0) + f"v_add_nc_u32 {rx}, {rx}, v{s+3}", # q += 1 if needed + # Correction for b < 0: If r >= -b (r > 0 and r + b >= 0), q is too high - decrement q + f"v_cmp_gt_i32 vcc_lo, v{s+4}, 0", # vcc = (r > 0) + f"v_cndmask_b32 v{s+3}, 0, 1, vcc_lo", # s+3 = 1 if r > 0 + f"v_add_nc_u32 v{s+5}, v{s+4}, v{s+1}", # s+5 = r + b (when b < 0, this checks r >= -b) + f"v_cmp_ge_i32 vcc_lo, v{s+5}, 0", # vcc = (r + b >= 0), i.e., r >= -b + f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", + f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0) + f"v_cndmask_b32 v{s+3}, 0, v{s+3}, vcc_lo", + f"v_sub_nc_u32 {rx}, {rx}, v{s+3}", # q -= 1 if needed ] def render_signed_small_int_mod(ctx, x, a, b): - """Render MOD for signed int8/int16. Sign-extend operands first, then compute a - (a/b)*b.""" + """Render MOD for signed int8/int16. Sign-extend operands first, compute corrected quotient, then remainder. + v_rcp_f32 is an approximation, so we need correction for both positive and negative divisors.""" dtype = x.dtype bits = 8 if dtype == dtypes.int8 else 16 ra, rb, rx = ctx.r[a], ctx.r[b], ctx.r[x] s = ctx.get_scratch_vgpr() - # Sign-extend and compute mod = a - (a/b)*b + # Use same correction logic as IDIV, then compute r = a - q*b + # s: sign-extended a, s+1: sign-extended b, s+2: 1/float(b), s+3: temp/q, s+4: temp, s+5: temp, s+6: temp return [ - f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # sign-extend a - f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # sign-extend b - f"v_cvt_f32_i32 v{s+2}, v{s}", # float(a) - f"v_cvt_f32_i32 v{s+3}, v{s+1}", # float(b) - f"v_rcp_f32 v{s+3}, v{s+3}", # 1/float(b) - f"v_mul_f32 v{s+2}, v{s+2}, v{s+3}", # float(a)/float(b) - f"v_trunc_f32 v{s+2}, v{s+2}", # truncate toward zero - f"v_cvt_i32_f32 v{s+2}, v{s+2}", # q = trunc(a/b) - f"v_mul_lo_u32 v{s+2}, v{s+2}, v{s+1}", # q*b - f"v_sub_nc_u32 {rx}, v{s}, v{s+2}", # a - q*b + f"v_bfe_i32 v{s}, {ra}, 0, {bits}", # s = sign-extend a + f"v_bfe_i32 v{s+1}, {rb}, 0, {bits}", # s+1 = sign-extend b + # Initial float division + f"v_cvt_f32_i32 v{s+3}, v{s}", # s+3 = float(a) + f"v_cvt_f32_i32 v{s+4}, v{s+1}", # s+4 = float(b) + f"v_rcp_f32 v{s+2}, v{s+4}", # s+2 = 1/float(b) (approx) + f"v_mul_f32 v{s+3}, v{s+3}, v{s+2}", # s+3 = float(a)/float(b) (approx) + f"v_trunc_f32 v{s+3}, v{s+3}", # s+3 = trunc(a/b) + f"v_cvt_i32_f32 v{s+3}, v{s+3}", # s+3 = q (initial quotient) + # Compute exact remainder r = a - q*b + f"v_mul_lo_u32 v{s+4}, v{s+3}, v{s+1}", # s+4 = q*b + f"v_sub_nc_u32 v{s+4}, v{s}, v{s+4}", # s+4 = r = a - q*b (exact remainder) + # First correction pass using float - r is now smaller, fits in float24 + f"v_cvt_f32_i32 v{s+5}, v{s+4}", # s+5 = float(r) + f"v_mul_f32 v{s+5}, v{s+5}, v{s+2}", # s+5 = float(r)/float(b) + f"v_trunc_f32 v{s+5}, v{s+5}", # s+5 = trunc(r/b) + f"v_cvt_i32_f32 v{s+5}, v{s+5}", # s+5 = large correction + f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += large correction + # Recompute remainder after large correction + f"v_mul_lo_u32 v{s+5}, v{s+3}, v{s+1}", # s+5 = q*b + f"v_sub_nc_u32 v{s+4}, v{s}, v{s+5}", # s+4 = r (after large correction) + # Correction for b > 0: If r >= b, q is too low - increment q + f"v_cmp_ge_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r >= b) + f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r >= b + f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0) + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", # s+5 = 1 if (r >= b AND b > 0) + f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += 1 if needed + # Correction for b > 0: If r <= -b (r < 0 and r + b <= 0), q is too high - decrement q + f"v_cmp_lt_i32 vcc_lo, v{s+4}, 0", # vcc = (r < 0) + f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r < 0 + f"v_add_nc_u32 v{s+6}, v{s+4}, v{s+1}", # s+6 = r + b + f"v_cmp_le_i32 vcc_lo, v{s+6}, 0", # vcc = (r + b <= 0) + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", + f"v_cmp_gt_i32 vcc_lo, v{s+1}, 0", # vcc = (b > 0) + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", + f"v_sub_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q -= 1 if needed + # Correction for b < 0: If r <= b (r is more negative), q is too low - increment q + f"v_cmp_le_i32 vcc_lo, v{s+4}, v{s+1}", # vcc = (r <= b) + f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r <= b + f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0) + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", # s+5 = 1 if (r <= b AND b < 0) + f"v_add_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q += 1 if needed + # Correction for b < 0: If r >= -b (r > 0 and r + b >= 0), q is too high - decrement q + f"v_cmp_gt_i32 vcc_lo, v{s+4}, 0", # vcc = (r > 0) + f"v_cndmask_b32 v{s+5}, 0, 1, vcc_lo", # s+5 = 1 if r > 0 + f"v_add_nc_u32 v{s+6}, v{s+4}, v{s+1}", # s+6 = r + b (when b < 0, this checks r >= -b) + f"v_cmp_ge_i32 vcc_lo, v{s+6}, 0", # vcc = (r + b >= 0), i.e., r >= -b + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", + f"v_cmp_lt_i32 vcc_lo, v{s+1}, 0", # vcc = (b < 0) + f"v_cndmask_b32 v{s+5}, 0, v{s+5}, vcc_lo", + f"v_sub_nc_u32 v{s+3}, v{s+3}, v{s+5}", # q -= 1 if needed + # Final remainder: r = a - q*b + f"v_mul_lo_u32 v{s+3}, v{s+3}, v{s+1}", # s+3 = q*b + f"v_sub_nc_u32 {rx}, v{s}, v{s+3}", # rx = a - q*b = final remainder ] def render_signed_small_int_mul(ctx, x, a, b): @@ -970,25 +1031,28 @@ def render_64bit_shl(ctx, x, a, b): def render_64bit_add(ctx, x, a, b): """Render 64-bit integer addition with carry: dst = a + b. Uses v_add_co_u32 for low 32 bits (produces carry), v_add_co_ci_u32 for high (consumes carry). - Handles constants: for small constants, high part is 0; for register pairs, extracts lo/hi.""" + Handles constants: for negative constants, high part is -1; for positive, high part is 0.""" rx = ctx.r[x] ra = ctx.r[a] rb = ctx.r[b] dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1 # Helper to extract lo/hi from a 64-bit operand (register pair or constant) - def get_lo_hi(r): + def get_lo_hi(r, uop=None): if '[' in r: # register pair like v[10:11] base = get_reg_base(r) return f"v{base}", f"v{base+1}" elif r.startswith('v') or r.startswith('s'): # single register num = int(r[1:]) return f"v{num}", f"v{num}" # same reg for both (shouldn't happen for 64-bit) - else: # constant - high part is 0 for small constants + else: # constant - high part is -1 for negative, 0 for positive + # Check if this is a negative constant by looking at the UOp arg + if uop is not None and uop.op is Ops.CONST and isinstance(uop.arg, int) and uop.arg < 0: + return r, "-1" # sign extend: high part is all 1s for negative return r, "0" - a_lo, a_hi = get_lo_hi(ra) - b_lo, b_hi = get_lo_hi(rb) + a_lo, a_hi = get_lo_hi(ra, a) + b_lo, b_hi = get_lo_hi(rb, b) return [ f"v_add_co_u32 v{dst_lo}, vcc_lo, {a_lo}, {b_lo}", # low + low, carry in vcc_lo @@ -998,25 +1062,28 @@ def render_64bit_add(ctx, x, a, b): def render_64bit_sub(ctx, x, a, b): """Render 64-bit integer subtraction with borrow: dst = a - b. Uses v_sub_co_u32 for low 32 bits (produces borrow), v_sub_co_ci_u32 for high (consumes borrow). - Handles constants: for small constants, high part is 0; for register pairs, extracts lo/hi.""" + Handles constants: for negative constants, high part is -1; for positive, high part is 0.""" rx = ctx.r[x] ra = ctx.r[a] rb = ctx.r[b] dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1 # Helper to extract lo/hi from a 64-bit operand (register pair or constant) - def get_lo_hi(r): + def get_lo_hi(r, uop=None): if '[' in r: # register pair like v[10:11] base = get_reg_base(r) return f"v{base}", f"v{base+1}" elif r.startswith('v') or r.startswith('s'): # single register num = int(r[1:]) return f"v{num}", f"v{num}" # same reg for both (shouldn't happen for 64-bit) - else: # constant - high part is 0 for small constants + else: # constant - high part is -1 for negative, 0 for positive + # Check if this is a negative constant by looking at the UOp arg + if uop is not None and uop.op is Ops.CONST and isinstance(uop.arg, int) and uop.arg < 0: + return r, "-1" # sign extend: high part is all 1s for negative return r, "0" - a_lo, a_hi = get_lo_hi(ra) - b_lo, b_hi = get_lo_hi(rb) + a_lo, a_hi = get_lo_hi(ra, a) + b_lo, b_hi = get_lo_hi(rb, b) return [ f"v_sub_co_u32 v{dst_lo}, vcc_lo, {a_lo}, {b_lo}", # low - low, borrow in vcc_lo @@ -1068,6 +1135,42 @@ def render_64bit_and(ctx, x, a, b): f"v_and_b32 v{dst_hi}, v{a_hi}, v{b_hi}", ] +def render_64bit_max(ctx, x, a, b): + """Render 64-bit integer MAX: dst = max(a, b). + Algorithm: Compare 64-bit values, then select larger one. + For signed: compare high parts as signed, low parts as unsigned when high parts equal. + For unsigned: compare both parts as unsigned.""" + rx = ctx.r[x] + ra = ctx.r[a] + rb = ctx.r[b] + dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1 + a_lo = get_reg_base(ra) if '[' in ra else int(ra[1:]) + a_hi = a_lo + 1 if '[' in ra else a_lo + b_lo = get_reg_base(rb) if '[' in rb else int(rb[1:]) + b_hi = b_lo + 1 if '[' in rb else b_lo + is_signed = x.dtype in (dtypes.int64, dtypes.long) + hi_cmp = "i32" if is_signed else "u32" + + # Use scratch SGPRs for comparison intermediates + ctx.scratch_sgpr_used = True + ss0, ss1 = f"s{ctx.gated_sgpr}", f"s{ctx.gated_sgpr+1}" + + # Compare a > b: (a_hi > b_hi) || (a_hi == b_hi && a_lo > b_lo) + # Then select: if a > b, choose a; else choose b + return [ + # Compare high parts (signed for signed types) + f"v_cmp_gt_{hi_cmp} vcc_lo, v{a_hi}, v{b_hi}", # vcc_lo = a_hi > b_hi + f"s_mov_b32 {ss0}, vcc_lo", # save hi_gt + f"v_cmp_eq_u32 vcc_lo, v{a_hi}, v{b_hi}", # vcc_lo = a_hi == b_hi + f"s_mov_b32 {ss1}, vcc_lo", # save hi_eq + f"v_cmp_gt_u32 vcc_lo, v{a_lo}, v{b_lo}", # vcc_lo = a_lo > b_lo (always unsigned) + f"s_and_b32 {ss1}, {ss1}, vcc_lo", # ss1 = hi_eq AND lo_gt + f"s_or_b32 vcc_lo, {ss0}, {ss1}", # vcc_lo = a > b (full 64-bit comparison) + # Select: if a > b, choose a; else choose b + f"v_cndmask_b32 v{dst_lo}, v{b_lo}, v{a_lo}, vcc_lo", + f"v_cndmask_b32 v{dst_hi}, v{b_hi}, v{a_hi}, vcc_lo", + ] + def render_64bit_udiv(ctx, x, a, b): """Render 64-bit unsigned division using f64 arithmetic. Algorithm: convert to f64, divide, floor, convert back, then refine. @@ -1252,7 +1355,8 @@ def render_64bit_udiv(ctx, x, a, b): ] def render_64bit_idiv(ctx, x, a, b): - """Render 64-bit signed division using f64 arithmetic.""" + """Render 64-bit signed division using f64 arithmetic. + Special cases: b==1 returns a, b==-1 returns -a (avoids f64 precision loss).""" rx, ra, rb = ctx.r[x], ctx.r[a], ctx.r[b] dst_lo, dst_hi = get_reg_base(rx), get_reg_base(rx) + 1 a_lo = get_reg_base(ra) if '[' in ra else int(ra[1:]) @@ -1262,6 +1366,19 @@ def render_64bit_idiv(ctx, x, a, b): s = ctx.get_scratch_vgpr() # For signed division, convert to unsigned, divide, then fix sign return [ + # Special case: if b == 1, result = a; if b == -1, result = -a + # Check: b_hi == 0 && b_lo == 1 means b == 1 + # Check: b_hi == -1 && b_lo == -1 means b == -1 (0xFFFFFFFFFFFFFFFF) + # Use s+20 and s+21 for flags to avoid conflict with f64 computation (uses s+0..s+16) + f"v_xor_b32 v{s+17}, v{b_lo}, 1", # b_lo XOR 1 (0 if b_lo == 1) + f"v_or_b32 v{s+17}, v{s+17}, v{b_hi}", # combined: 0 if b == 1 + f"v_cmp_eq_u32 vcc_lo, v{s+17}, 0", # vcc = (b == 1) + f"v_cndmask_b32 v{s+20}, 0, 1, vcc_lo", # s+20 = b_is_one flag + # Check for b == -1 (all bits set) + f"v_and_b32 v{s+17}, v{b_lo}, v{b_hi}", # AND of both halves + f"v_cmp_eq_i32 vcc_lo, v{s+17}, -1", # vcc = both halves are -1 + f"v_cndmask_b32 v{s+21}, 0, 1, vcc_lo", # s+21 = b_is_neg_one flag + # Get absolute values and track signs # sign_a = a >> 63, sign_b = b >> 63 f"v_ashrrev_i32 v{s}, 31, v{a_hi}", # s = sign extension of a @@ -1320,11 +1437,298 @@ def render_64bit_idiv(ctx, x, a, b): f"v_fma_f64 v[{s+9}:{s+10}], v[{s+9}:{s+10}], v[{s+11}:{s+12}], v[{s+7}:{s+8}]", # q_lo = q + q_hi*(-2^32) f"v_cvt_u32_f64 v{s+3}, v[{s+9}:{s+10}]", # q_lo as u32 + # Refinement: compute remainder and adjust q if needed (for values > 2^53 where f64 loses precision) + # abs_a is still recoverable from a and sign (s), abs_b is in s+5:s+6 + # Recompute abs_a from original a: abs_a = (a ^ sign) - sign + f"v_xor_b32 v{s+7}, v{a_lo}, v{s}", + f"v_xor_b32 v{s+8}, v{a_hi}, v{s}", + f"v_sub_co_u32 v{s+7}, vcc_lo, v{s+7}, v{s}", + f"v_sub_co_ci_u32 v{s+8}, vcc_lo, v{s+8}, v{s}, vcc_lo", # abs_a in s+7:s+8 + + # Refinement pass 1: compute product = q * abs_b (lower 64 bits) + # product = q_lo*b_lo + (q_lo*b_hi + q_hi*b_lo) << 32 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", # product_lo = low(q_lo * b_lo) + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", # tmp = high(q_lo * b_lo) + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", # q_lo * b_hi (contributes to product_hi) + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", # q_hi * b_lo (contributes to product_hi) + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", # product_hi = tmp + q_lo*b_hi + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", # product_hi += q_hi*b_lo + # Compute remainder = abs_a - product + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", # r_lo = abs_a_lo - product_lo + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", # r_hi + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", # borrow flag (1 = q too high) + # If borrow, decrement q + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + # If no borrow, check if r >= b and increment q + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", # r_hi > b_hi? + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", # r_hi == b_hi? + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", # r_lo >= b_lo? + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", # (r_hi == b_hi) && (r_lo >= b_lo) + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", # r >= b + f"v_xor_b32 v{s+15}, v{s+13}, 1", # no_borrow = !borrow + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", # increment only if no_borrow && r >= b + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + + # Refinement pass 2 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + + # Refinement pass 3 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + + # Refinement passes 4-7 (for very large values with f64 error up to ~10) + # Pass 4 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Pass 5 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Pass 6 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Pass 7 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Pass 8 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", + f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", + f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", + f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", + f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", + f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", + f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", + f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", + f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", + f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", + f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", + f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_xor_b32 v{s+15}, v{s+13}, 1", + f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", + f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", + f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", + f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", + f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Passes 9-15 (abbreviated for brevity - same pattern) + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", f"v_xor_b32 v{s+15}, v{s+13}, 1", f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Pass 10 + f"v_mul_lo_u32 v{s+9}, v{s+3}, v{s+5}", f"v_mul_hi_u32 v{s+10}, v{s+3}, v{s+5}", f"v_mul_lo_u32 v{s+11}, v{s+3}, v{s+6}", f"v_mul_lo_u32 v{s+12}, v{s+4}, v{s+5}", + f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+11}", f"v_add_co_u32 v{s+10}, vcc_lo, v{s+10}, v{s+12}", + f"v_sub_co_u32 v{s+11}, vcc_lo, v{s+7}, v{s+9}", f"v_sub_co_ci_u32 v{s+12}, vcc_lo, v{s+8}, v{s+10}, vcc_lo", f"v_cndmask_b32 v{s+13}, 0, 1, vcc_lo", + f"v_sub_co_u32 v{s+14}, vcc_lo, v{s+3}, v{s+13}", f"v_sub_co_ci_u32 v{s+15}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+13}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+14}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+15}, vcc_lo", + f"v_cmp_gt_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+14}, 0, 1, vcc_lo", f"v_cmp_eq_u32 vcc_lo, v{s+12}, v{s+6}", f"v_cndmask_b32 v{s+15}, 0, 1, vcc_lo", f"v_cmp_ge_u32 vcc_lo, v{s+11}, v{s+5}", f"v_cndmask_b32 v{s+16}, 0, 1, vcc_lo", + f"v_and_b32 v{s+15}, v{s+15}, v{s+16}", f"v_or_b32 v{s+14}, v{s+14}, v{s+15}", f"v_xor_b32 v{s+15}, v{s+13}, 1", f"v_and_b32 v{s+14}, v{s+14}, v{s+15}", + f"v_add_co_u32 v{s+15}, vcc_lo, v{s+3}, v{s+14}", f"v_add_co_ci_u32 v{s+16}, vcc_lo, v{s+4}, 0, vcc_lo", f"v_cmp_ne_u32 vcc_lo, v{s+14}, 0", f"v_cndmask_b32 v{s+3}, v{s+3}, v{s+15}, vcc_lo", f"v_cndmask_b32 v{s+4}, v{s+4}, v{s+16}, vcc_lo", + # Apply sign: result = (abs_q ^ result_sign) - result_sign f"v_xor_b32 v{dst_lo}, v{s+3}, v{s+2}", f"v_xor_b32 v{dst_hi}, v{s+4}, v{s+2}", f"v_sub_co_u32 v{dst_lo}, vcc_lo, v{dst_lo}, v{s+2}", f"v_sub_co_ci_u32 v{dst_hi}, vcc_lo, v{dst_hi}, v{s+2}, vcc_lo", + + # Special case: if b == 1, use a directly + f"v_cmp_ne_u32 vcc_lo, v{s+20}, 0", # vcc = (b == 1) + f"v_cndmask_b32 v{dst_lo}, v{dst_lo}, v{a_lo}, vcc_lo", # dst_lo = a_lo if b==1 else q_lo + f"v_cndmask_b32 v{dst_hi}, v{dst_hi}, v{a_hi}, vcc_lo", # dst_hi = a_hi if b==1 else q_hi + + # Special case: if b == -1, use -a (negate a) + f"v_xor_b32 v{s+3}, v{a_lo}, -1", # ~a_lo + f"v_xor_b32 v{s+4}, v{a_hi}, -1", # ~a_hi + f"v_add_co_u32 v{s+3}, vcc_lo, v{s+3}, 1", # neg_a_lo = ~a_lo + 1 + f"v_add_co_ci_u32 v{s+4}, vcc_lo, v{s+4}, 0, vcc_lo", # neg_a_hi = ~a_hi + carry + f"v_cmp_ne_u32 vcc_lo, v{s+21}, 0", # vcc = (b == -1) + f"v_cndmask_b32 v{dst_lo}, v{dst_lo}, v{s+3}, vcc_lo", # dst_lo = neg_a_lo if b==-1 + f"v_cndmask_b32 v{dst_hi}, v{dst_hi}, v{s+4}, vcc_lo", # dst_hi = neg_a_hi if b==-1 ] def render_64bit_umod(ctx, x, a, b): @@ -1594,6 +1998,9 @@ string_rewrite = PatternMatcher([ # signed int8/int16 MAX need sign extension (MIN uses MAX + XOR pattern) (UPat(Ops.MAX, name="x", dtype=dtypes.int8, src=(UPat.var("a"), UPat.var("b"))), render_signed_small_int_max), (UPat(Ops.MAX, name="x", dtype=dtypes.int16, src=(UPat.var("a"), UPat.var("b"))), render_signed_small_int_max), + # 64-bit integer MAX: need full 64-bit comparison + (UPat(Ops.MAX, name="x", dtype=dtypes.long, src=(UPat.var("a"), UPat.var("b"))), render_64bit_max), + (UPat(Ops.MAX, name="x", dtype=dtypes.ulong, src=(UPat.var("a"), UPat.var("b"))), render_64bit_max), # alu ops - extract low 32 bits for 64-bit integer operands since RDNA has limited 64-bit int ALU (UPat(GroupOp.ALU, name="x"), lambda ctx, x: ctx.code_for_op[x.op]( extract_low_32(ctx.r[x]) if x.dtype in (dtypes.long, dtypes.ulong) else ctx.r[x], @@ -3215,11 +3622,13 @@ amdhsa.version: if recompute_const is not None and recompute_base is not None: # ADD(base, const) - use v_lshl_add_u32 to compute (base << shift) + (const << shift) base_reg = r.get(recompute_base) - if base_reg and isinstance(base_reg, str) and base_reg.startswith('v'): + # v_lshl_add_u32 accepts both VGPR and SGPR sources + if base_reg and isinstance(base_reg, str) and (base_reg.startswith('v') or base_reg.startswith('s')): kernel.append(f"v_lshl_add_u32 {deferred_store_addr_vgpr}, {base_reg}, {shift_val}, {recompute_const << shift_val}") else: - # Fallback: copy from pre-computed - kernel.append(f"v_mov_b32 {deferred_store_addr_vgpr}, {r[idx]}") + # Fallback: compute the SHL manually if we can't recompute inline + # This shouldn't normally happen, but handle gracefully + kernel.append(f"v_mov_b32 {deferred_store_addr_vgpr}, {r[idx] if r.get(idx) != 'RECOMPUTE_AT_STORE' else 0}") else: # ADD(reg, reg) - compute ADD then SHL kernel.append(f"v_add_nc_u32 {deferred_store_addr_vgpr}, {r[add_src0]}, {r[add_src1]}")