diff --git a/test/null/test_simplify_valid_idx.py b/test/null/test_simplify_valid_idx.py index 1fd781651a..97b23e7635 100644 --- a/test/null/test_simplify_valid_idx.py +++ b/test/null/test_simplify_valid_idx.py @@ -21,7 +21,7 @@ def get_gated_load_uop(valid:UOp, idx:UOp): def get_load_image_uop(image_shape:tuple[int, ...], valid:UOp, idx:tuple[UOp, UOp]): return UOp(Ops.LOAD, dtypes.float.vec(4), ( - UOp(Ops.PARAM, dtypes.imagef(image_shape), arg=0).index(UOp(Ops.STACK, dtypes.weakint.vec(2), idx).valid(valid), ptr=True), + UOp(Ops.PARAM, dtypes.imagef(image_shape), arg=0).index(idx[1].valid(valid), idx[0].valid(valid), ptr=True), UOp(Ops.STACK, dtypes.float.vec(4), src=(UOp.const(dtypes.float, 0.0),) * 4) )) @@ -222,17 +222,16 @@ class TestValidIdxSimplification(unittest.TestCase): class TestImageSimplification(unittest.TestCase): def check(self, load, svalid, sidx0, sidx1): load = simplify_image_idx(load.sink()).src[0] - off = load.src[0].src[1] - idx = off.get_idx() - self.assertEqual(idx.op, Ops.STACK) - self.assertEqual(len(idx.src), 2) - idx0, idx1 = idx.src[0], idx.src[1] + off = load.src[0] + self.assertEqual(len(off.src), 3) + idx0, idx1 = off.src[2].get_idx(), off.src[1].get_idx() check_uop_against_string(self, idx0, sidx0) check_uop_against_string(self, idx1, sidx1) + self.assertEqual(off.src[1].get_valid(), off.src[2].get_valid()) if svalid is not None: - check_uop_against_string(self, off.get_valid(), svalid) + check_uop_against_string(self, off.src[1].get_valid(), svalid) else: - self.assertEqual(off.get_valid(), UOp.const(dtypes.bool, True), "svalid is None but valid is not True") + self.assertEqual(off.src[1].get_valid(), UOp.const(dtypes.bool, True), "svalid is None but valid is not True") def test_idx_gt_c(self): # (idx1 < c+1).ne(True) ? (..., idx1-1+c) : 0 can drop the valid diff --git a/tinygrad/codegen/late/devectorizer.py b/tinygrad/codegen/late/devectorizer.py index 871c22e758..5fb7403a5a 100644 --- a/tinygrad/codegen/late/devectorizer.py +++ b/tinygrad/codegen/late/devectorizer.py @@ -38,21 +38,24 @@ def _drop_valid_stmts(valid:UOp, idx:UOp, height:int, width:int) -> list[UOp]: def simplify_valid_load(buf:UOp, start_idx:UOp, valid:UOp) -> UOp|None: idx = uop_given_valid(valid, start_idx) - if not isinstance(buf.dtype, ImageDType): return None if idx is start_idx else buf.index(idx.valid(valid), ptr=True) - - # wait for it to be image indexed before running simplification - if start_idx.dtype.count != 2: return None + return None if idx is start_idx else buf.index(idx.valid(valid), ptr=True) +def simplify_valid_image_load(buf:UOp, idx_y:UOp, idx_x:UOp, valid:UOp) -> UOp|None: + if not isinstance(buf.dtype, ImageDType): return None + start_idx = UOp.vectorize(idx_x, idx_y) + idx = uop_given_valid(valid, start_idx) drop_stmt = _drop_valid_stmts(valid, idx, buf.dtype.shape[0], buf.dtype.shape[1]) if not drop_stmt and idx is start_idx: return None new_valid = UOp.uprod(*ss) if (ss:=[s for s in valid.split_uop(Ops.AND) if s not in drop_stmt]) else None - return buf.index(idx.valid(new_valid) if new_valid is not None else idx, ptr=True) - + idx_y, idx_x = idx.gep(1), idx.gep(0) + return buf.index(idx_y.valid(new_valid), idx_x.valid(new_valid), ptr=True) if new_valid is not None else buf.index(idx_y, idx_x, ptr=True) load_store_indexing = PatternMatcher([ # image load valid idx simplification (UPat(Ops.INDEX, src=(UPat.var("buf"), invalid_gate)), lambda buf,x,i,cond: simplify_valid_load(buf, x, cond)), + (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("valid").where(UPat.var("idx_y"), UPat(arg=Invalid)), + UPat.var("valid").where(UPat.var("idx_x"), UPat(arg=Invalid)))), simplify_valid_image_load), ]) # ***** load/store grouping ***** @@ -195,7 +198,7 @@ def split_load_store(ctx:Renderer|None, ls:UOp, idx:UOp): def get_image_idx(idx:UOp, width:int): x, valid = idx.src[1].get_idx(), idx.src[1].get_valid() idx_x, idx_y = (x // 4) % width, x // (4*width) - return idx.replace(src=(idx.src[0], UOp.vectorize(idx_x, idx_y).valid(valid))) + return idx.replace(src=(idx.src[0], idx_y.valid(valid), idx_x.valid(valid))) def image_fixup(ls:UOp): # normal image load or store, with the CAST from expand_index @@ -204,7 +207,8 @@ def image_fixup(ls:UOp): return ls.replace(src=(get_image_idx(ls.src[0].src[0], dt.shape[1]),)+ls.src[1:]) # this is an unprocessed image without a cast, we should just make it a buffer - if isinstance(dt, ImageDType) and (off:=ls.src[0].src[1]).get_idx().dtype != dtypes.weakint.vec(2): + if isinstance(dt, ImageDType) and len(ls.src[0].src) == 2: + off = ls.src[0].src[1] idx = ls.src[0].src[0].replace(dtype=(new_dt:=dtypes.half if dt.itemsize == 2 else dtypes.float).ptr(dt.size)).index(off) return ls.replace(src=(idx,), dtype=new_dt).cast(dtypes.float) if ls.op is Ops.LOAD else ls.replace(src=(idx, ls.src[1].cast(new_dt))) diff --git a/tinygrad/codegen/late/gater.py b/tinygrad/codegen/late/gater.py index a2b96f175b..2f2195bd96 100644 --- a/tinygrad/codegen/late/gater.py +++ b/tinygrad/codegen/late/gater.py @@ -9,13 +9,17 @@ pm_move_gates_from_index = PatternMatcher([ (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx"), UPat(arg=Invalid))).or_casted(name="cast").store(UPat.var("data")), lambda buf,gate,idx,cast,data: buf.index(idx, ptr=True).cast(cast.dtype).store(data, gate)), + # for image idx + (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), + UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).or_casted(name="cast").load(name="l"), + lambda buf,gate,idx_y,idx_x,cast,l: buf.index(idx_y, idx_x, ptr=True).cast(cast.dtype).load(l.const_like(0), gate, dtype=l.dtype)), + (UPat.var("buf").index(UPat.var("gate").where(UPat.var("idx_y"), UPat(arg=Invalid)), + UPat.var("gate").where(UPat.var("idx_x"), UPat(arg=Invalid))).or_casted(name="cast").store(UPat.var("data")), + lambda buf,gate,idx_y,idx_x,cast,data: buf.index(idx_y, idx_x, ptr=True).cast(cast.dtype).store(data, gate)), + # Where after gated load becomes alt value (UPat.var("gate").where(UPat().load(UPat(), UPat.var("gate", dtype=dtypes.bool), name="l").or_casted(), UPat.var("a")), lambda gate,l,a: l.replace(src=(l.src[0], a.src[0] if a.op is Ops.CAST and a.src[0].dtype == l.dtype else a.cast(l.dtype), l.src[2])).cast(a.dtype)), (UPat.var("gate").where(UPat.var("a"), UPat().load(UPat(), ~UPat.var("gate", dtype=dtypes.bool), name="l").or_casted()), lambda gate,l,a: l.replace(src=(l.src[0], a.src[0] if a.op is Ops.CAST and a.src[0].dtype == l.dtype else a.cast(l.dtype), l.src[2])).cast(a.dtype)), - - # images use 2D INDEX now (y,x) - (UPat(Ops.INDEX, src=(UPat(), UPat((Ops.CONST, Ops.STACK), name="vec")), name="idx"), - lambda idx,vec: idx.replace(src=(idx.src[0], vec.gep(1).cast(dtypes.int), vec.gep(0).cast(dtypes.int))) if vec.dtype.count == 2 else None), ]) diff --git a/tinygrad/renderer/nir.py b/tinygrad/renderer/nir.py index 1f2cb5c460..f7ed09fc55 100644 --- a/tinygrad/renderer/nir.py +++ b/tinygrad/renderer/nir.py @@ -139,6 +139,9 @@ class NIRRenderer(Renderer): (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("off")), name="x"), lambda x,buf,off: x.replace( src=(buf,off.cast(dtypes.long))) if buf.dtype.addrspace != AddrSpace.REG and off.op not in (Ops.CAST, Ops.STACK) else None), (UPat(Ops.CAST, name="x"), lambda x: x.src[0] if isinstance(x.dtype, PtrDType) or x.src[0].dtype == dtypes.void else None), + # images need index to be int for nir + (UPat.var("buf").index(UPat.var("idx_y"), UPat.var("idx_x")), + lambda buf,idx_y,idx_x: buf.index(idx_y.cast(dtypes.int), idx_x.cast(dtypes.int))), ]) def_rewrite = PatternMatcher([ diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index e280554e88..083cc6cc38 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -1589,6 +1589,14 @@ pm_lower_index_dtype = PatternMatcher([ (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("idx", dtypes.ints).cast()),), lambda buf,idx: buf.index(idx, ptr=True)), (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("gate").where(UPat.var("idx", dtypes.ints).cast(), UPat(Ops.CONST, arg=Invalid)))), lambda buf,idx,gate: buf.index(gate.where(idx, idx.const_like(Invalid)), ptr=True)), + # remove hanging casts for images + (UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var("idx_y", dtypes.ints).cast(), UPat.var("idx_x", dtypes.ints).cast()),), + lambda buf,idx_x,idx_y: buf.index(idx_y, idx_x, ptr=True)), + (UPat(Ops.INDEX, src=(UPat.var("buf"), + UPat.var("gate").where(UPat.var("idx_y", dtypes.ints).cast(), UPat(Ops.CONST, arg=Invalid)), + UPat.var("gate").where(UPat.var("idx_x", dtypes.ints).cast(), UPat(Ops.CONST, arg=Invalid)))), + lambda buf,idx_x,idx_y,gate: buf.index(gate.where(idx_y, idx_y.const_like(Invalid)), + gate.where(idx_x, idx_x.const_like(Invalid)), ptr=True)), (UPat((Ops.SINK, Ops.NOOP, Ops.END), name="n"), lambda n: n.replace(src=tuple(s.src[0] if s.op is Ops.CAST and s.dtype == dtypes.weakint else s for s in n.src))), ]) diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index de6099c604..ac0bacd77e 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -81,9 +81,9 @@ propagate_invalid = PatternMatcher([ (UPat(Ops.BITCAST, src=(invalid_pat,), name="bc"), lambda bc,i: i.cast(bc.dtype)), (UPat(Ops.BITCAST, src=(invalid_gate,), name="bc"), lambda bc,cond,x,i: cond.where(x.bitcast(bc.dtype), i.bitcast(bc.dtype))), # fold gated LOAD/STORE - (UPat(Ops.STORE, src=(UPat().index(invalid_pat).or_casted(), UPat())), lambda i: UOp(Ops.NOOP)), - (UPat(Ops.LOAD, src=(UPat().index(invalid_pat).or_casted(),), allow_any_len=True, name="x"), - lambda x,i: x.src[1] if len(x.src) > 1 else x.const_like(0)), # invalid load produces 0, or the alt value if we have one + (UPat(Ops.STORE, src=(UPat(Ops.INDEX, src=(UPat(), invalid_pat), allow_any_len=True).or_casted(), UPat())), lambda i: UOp(Ops.NOOP)), + (UPat(Ops.LOAD, src=(UPat(Ops.INDEX, src=(UPat(), invalid_pat), allow_any_len=True).or_casted(),), allow_any_len=True, name="x"), + lambda x,i: x.src[1] if len(x.src) > 1 else x.const_like(0)), ]) symbolic_simple = propagate_invalid + PatternMatcher([