Revert "remove the on_stack stuff and any retries"

This reverts commit 49a2b328b9.

test/test_rangeify.py

@@ -300,11 +300,12 @@ class TestOuterworld(unittest.TestCase):
     o.contiguous(i).realize()
     self.assertTrue((t==o).all().item())
 
-from tinygrad.schedule.rangeify import pm_rangeify, RangeifyContext
+@unittest.skip("pm_rangeify no longer exists. test this in a different way")
 class TestRangeifyPM(unittest.TestCase):
   def setUp(self): self.base = Tensor.empty(10*10).reshape(10, 10).contiguous()
   def assert_same(self, a, b):
     def run_pm_rangeify(t:Tensor):
+      from tinygrad.schedule.rangeify import pm_rangeify, RangeifyContext
       sink = t.uop.sink()
       pm_realize = PatternMatcher([(UPat(Ops.CONTIGUOUS, name="x"), lambda x: x.replace(op=Ops.REALIZE))])
       sink = graph_rewrite(sink, pm_realize)

test/unit/test_rewrite_not_ready.py

@@ -1,110 +0,0 @@
-import unittest
-from dataclasses import dataclass, field
-from tinygrad.uop.ops import PatternMatcher, UOp, graph_rewrite, Ops, UPat, GroupOp, RewriteNotReady
-
-# we could insert CHILDREN node
-
-@dataclass
-class ChildrenContext:
-  children: dict[UOp, list[UOp]]|None = None
-
-# this is a generic child labeller
-def extract_children(ctx:ChildrenContext, x:UOp):
-  if ctx.children is not None: return
-  ctx.children = {k:list(v.keys()) for k,v in x.get_consumer_map().items() if len(v) > 1}
-
-def mark_children(ctx:ChildrenContext, x:UOp):
-  new_srcs = [(UOp(Ops.CHILD, s.dtype, src=(s,), arg=(ctx.children[s].index(x), len(ctx.children[s]))) if s in ctx.children else s) for s in x.src]
-  return x.replace(src=tuple(new_srcs))
-
-pm_children = PatternMatcher([
-  (UPat(Ops.SINK, name="x"), extract_children),
-  (UPat(GroupOp.All-{Ops.CHILD}, name="x"), mark_children),
-])
-
-@dataclass
-class TestContext:
-  seen_children: dict[UOp, set[int]] = field(default_factory=dict)
-  ready_children: dict[UOp, set[int]] = field(default_factory=dict)
-  seen_consts:int = 0
-  saved_seen_consts:int = 0
-  exp2_visit_count:int = 0
-
-# this is a generic pattern
-def visit_child(ctx:ChildrenContext, x:UOp):
-  if x.src[0] not in ctx.seen_children:
-    ctx.seen_children[x.src[0]] = set()
-    ctx.ready_children[x.src[0]] = set()
-  ctx.seen_children[x.src[0]].add(x.arg[0])
-  if len(ctx.seen_children[x.src[0]]) != x.arg[1]:
-    print(f"visit CHILD {x.arg} bottom up -- not ready {ctx.seen_children[x.src[0]]}")
-    raise RewriteNotReady
-  print(f"visit CHILD {x.arg} bottom up -- READY {ctx.seen_children[x.src[0]]}")
-  ctx.ready_children[x.src[0]].add(x.arg[0])
-
-pm_child_visitor = PatternMatcher([
-  (UPat(Ops.CHILD, name="x"), visit_child),
-])
-
-# this is for the test
-def see_const(ctx:ChildrenContext, c:UOp): ctx.seen_consts += c.arg
-def see_exp2(ctx:ChildrenContext): ctx.exp2_visit_count += 1
-def save_seen_consts(ctx:ChildrenContext, x:UOp): ctx.saved_seen_consts = ctx.seen_consts
-pm_consts = PatternMatcher([
-  (UPat(Ops.DEFINE_VAR, name="x"), save_seen_consts),
-  (UPat()+UPat.cvar("c"), see_const),
-  (UPat(Ops.EXP2), see_exp2),
-])
-
-class TestChildrenRewrite(unittest.TestCase):
-  def test_not_ready_double_simple(self):
-    global_a = UOp.variable("a", 0, 10).exp2()
-    inter = (global_a+global_a).exp2()
-    global_sink = (inter+inter).sink()
-
-    sink = graph_rewrite(global_sink, pm_children, ctx=ChildrenContext(), bottom_up=True)
-    ctx = TestContext()
-    graph_rewrite(sink, pm_consts, ctx=ctx, bottom_up=True)
-    self.assertEqual(ctx.exp2_visit_count, 2)
-
-  def test_not_ready_double(self):
-    global_a = UOp.variable("a", 0, 10).exp2()
-    inter = ((global_a+1000)+(global_a+100)).exp2()
-    global_sink = ((inter+10)+(inter+1)).sink()
-
-    sink = graph_rewrite(global_sink, pm_children, ctx=ChildrenContext(), bottom_up=True)
-    print("test_not_ready_double")
-    ctx = TestContext()
-    graph_rewrite(sink, pm_child_visitor+pm_consts, ctx=ctx, bottom_up=True)
-    self.assertEqual(ctx.exp2_visit_count, 2)
-    self.assertEqual(ctx.seen_consts, ctx.saved_seen_consts)
-    self.assertEqual(ctx.seen_consts, 1111)
-
-  def test_in_srcs_twice(self):
-    global_a = UOp.variable("a", 0, 10).exp2()
-    global_sink = (global_a+global_a).sink()
-
-    ctx = TestContext()
-    graph_rewrite(global_sink, pm_consts, ctx=ctx, bottom_up=True)
-    self.assertEqual(ctx.exp2_visit_count, 1)
-
-  def test_not_ready(self):
-    global_a = UOp.variable("a", 0, 10).exp2()
-    global_sink = ((global_a+2)+(global_a+3)).sink()
-
-    # without children and not ready, we don't see both adds before the DEFINE_VAR
-    ctx = TestContext()
-    graph_rewrite(global_sink, pm_consts, ctx=ctx, bottom_up=True)
-    self.assertNotEqual(ctx.seen_consts, ctx.saved_seen_consts)
-    self.assertEqual(ctx.exp2_visit_count, 1)
-
-    # with children and not ready we do
-    sink = graph_rewrite(global_sink, pm_children, ctx=ChildrenContext(), bottom_up=True)
-    ctx = TestContext()
-    graph_rewrite(sink, pm_child_visitor+pm_consts, ctx=ctx, bottom_up=True)
-    self.assertEqual(ctx.seen_consts, ctx.saved_seen_consts)
-    self.assertEqual(ctx.exp2_visit_count, 1)
-    self.assertSetEqual(list(ctx.ready_children.values())[0], {0,1})
-
-if __name__ == '__main__':
-  unittest.main()

tinygrad/schedule/indexing.py

@@ -7,6 +7,32 @@ from tinygrad.uop.symbolic import sym
 from tinygrad.helpers import argsort, all_same, Context
 from tinygrad.uop.ops import graph_rewrite, sint, AxisType
 
+ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.ASSIGN, Ops.COPY, Ops.BUFFER, Ops.BUFFER_VIEW,
+                     Ops.CONST, Ops.BIND, Ops.DEVICE, Ops.MSELECT, Ops.MSTACK, Ops.DEFINE_GLOBAL,
+                     Ops.DEFINE_LOCAL, Ops.DEFINE_REG, Ops.LOAD, Ops.KERNEL}
+
+def realize(ctx:dict[UOp, None], tr:UOp) -> None: ctx[tr] = None
+
+def realize_srcs(ctx:dict[UOp, None], rb:UOp) -> None:
+  for s in rb.src:
+    if s.base.op not in ALWAYS_CONTIGUOUS: ctx[s] = None
+
+def realize_assign(ctx:dict[UOp, None], a:UOp) -> None:
+  if a.src[1].op not in ALWAYS_CONTIGUOUS: ctx[a.src[1]] = None
+  # if it's a kernel, we don't realize it
+  if a.src[1].op is not Ops.KERNEL: ctx[a] = None
+
+pm_generate_realize_map = PatternMatcher([
+  # always realize SINK src
+  (UPat(Ops.SINK, name="s"), lambda ctx,s: ctx.update((x.base, None) for x in s.src if x.base.op not in ALWAYS_CONTIGUOUS)),
+  # always realize ASSIGN/COPY/BUFFER_VIEW/CONTIGUOUS
+  (UPat({Ops.COPY, Ops.BUFFER_VIEW, Ops.CONTIGUOUS}, name="tr"), realize),
+  # realize srcs of COPY, MSELECT, MSTACK
+  (UPat((Ops.COPY, Ops.MSELECT, Ops.MSTACK), name="rb"), realize_srcs),
+  # realize input to assign (might be optimized out)
+  (UPat(Ops.ASSIGN, name="a"), realize_assign),
+])
+
 @dataclass(frozen=True)
 class BufferizeOpts:
   # on AddrSpace.LOCAL, device is the id
@@ -77,11 +103,15 @@ pm_apply_rangeify = PatternMatcher([
   (UPat((Ops.CONST, Ops.DEFINE_VAR), name="c"), lambda ctx,c: c.replace(src=()) if c in ctx.range_map else None),
 ])
 
-def run_rangeify(tsink:UOp, realize_map:dict[UOp, None], debug) -> tuple[UOp, IndexingContext]:
+def run_rangeify(tsink:UOp, debug:bool=False) -> tuple[UOp, IndexingContext]:
   tsink_base = UOp.sink(*[x.base for x in tsink.src])
 
-  # explicit rangeify
   rctx = IndexingContext()
+
+  # get ops to realize
+  graph_rewrite(tsink, pm_generate_realize_map, ctx=rctx.realize_map, name="Input Graph")
+
+  # explicit rangeify
   ending_ranges: dict[UOp, bool] = {}
   for x in tsink_base.reverse_toposort(consumer_map:=tsink_base.get_consumer_map()):
     if x.op in {Ops.DEVICE, Ops.UNIQUE}: continue
@@ -89,9 +119,9 @@ def run_rangeify(tsink:UOp, realize_map:dict[UOp, None], debug) -> tuple[UOp, In
 
     # if this element has weight and it's ending a range, we (force) realize it
     if ending_ranges[x] and x.op in GroupOp.Elementwise.union({Ops.REDUCE_AXIS}):
-      if x.op_in_backward_slice_with_self(Ops.BUFFER, Ops.REALIZE, Ops.BUFFERIZE, Ops.CONTIGUOUS):
+      if x.op_in_backward_slice_with_self(Ops.BUFFER, Ops.BUFFERIZE, Ops.CONTIGUOUS):
         if x.op_in_backward_slice_with_self(Ops.REDUCE_AXIS):
-          realize_map[x] = None
+          rctx.realize_map[x] = None
 
     # *** the ranges on the output are
     #  1. new if this op is realized
@@ -99,7 +129,7 @@ def run_rangeify(tsink:UOp, realize_map:dict[UOp, None], debug) -> tuple[UOp, In
     #  3. potentially new if this op has 2+ consumers
 
     consumer_rngs = [rctx.range_map[c][0] for c in consumer_map[x] if c in rctx.range_map]
-    if x in realize_map:
+    if x in rctx.realize_map:
       # if this is in the realize_map, we create new ranges (at the output)
       out_rngs = [rctx.new_range(s) for s in x.shape]
       # all ranges are ended now
@@ -136,7 +166,7 @@ def run_rangeify(tsink:UOp, realize_map:dict[UOp, None], debug) -> tuple[UOp, In
           out_rngs.append(rctx.new_range(x.shape[i]))
 
       # we have to realize here if there's new ranges
-      if not all_all_same: realize_map[x] = None
+      if not all_all_same: rctx.realize_map[x] = None
 
     # TODO: some ops don't have shape, enable this after the `.st` property is removed
     #assert len(out_rngs) == len(x.shape), \
@@ -190,12 +220,11 @@ def run_rangeify(tsink:UOp, realize_map:dict[UOp, None], debug) -> tuple[UOp, In
         if i in x.arg[1]: rngs[i] = rctx.new_range(s, axistype=AxisType.REDUCE)
 
     if debug:
-      print("***" if x in realize_map else "   ", len(consumer_map[x]), f"{str(x.op):20s}",
+      print("***" if x in rctx.realize_map else "   ", len(consumer_map[x]), f"{str(x.op):20s}",
             UOp.sink().index(*rngs).render(), " -> ", UOp.sink().index(*out_rngs).render())
 
     # assign to the range map. rngs are the input ranges, out_rngs are the output ranges, from the x op.
     rctx.range_map[x] = (rngs, out_rngs)
 
-  rctx.realize_map = realize_map
   tsink = graph_rewrite(tsink, pm_apply_rangeify, ctx=rctx, bottom_up=True, name="apply rangeify")
   return tsink, rctx

tinygrad/schedule/rangeify.py

@@ -1,16 +1,14 @@
-from typing import Any, cast, Iterator
-
-import functools, operator, itertools
+from typing import cast
 from dataclasses import dataclass, field
 from tinygrad.dtype import dtypes, PtrDType, ImageDType, AddrSpace
-from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, RewriteNotReady, _substitute, ssimplify, KernelInfo
+from tinygrad.uop.ops import PatternMatcher, UPat, Ops, UOp, resolve, GroupOp, _substitute, ssimplify, KernelInfo
 from tinygrad.uop.symbolic import sym, symbolic_simple
 from tinygrad.helpers import argsort, prod, all_same, pluralize, getenv, RANGEIFY, Context, flatten, dedup, unwrap, all_int, DEBUG, SPLIT_REDUCEOP
 from tinygrad.helpers import Metadata
 from tinygrad.uop.ops import track_rewrites, graph_rewrite, identity_element, sint, AxisType
 from tinygrad.codegen.simplify import pm_flatten_range, pm_reduce_unparented
 from tinygrad.codegen.opt import Opt
-from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, IndexingContext
+from tinygrad.schedule.indexing import run_rangeify, BufferizeOpts, ALWAYS_CONTIGUOUS, IndexingContext
 
 # creation can recurse a lot
 import sys
@@ -19,10 +17,6 @@ sys.setrecursionlimit(10000)
 # *****************
 # 0. do some cleanup rewrites, mostly copied from the old stuff
 
-ALWAYS_CONTIGUOUS: set[Ops] = {Ops.CONTIGUOUS, Ops.ASSIGN, Ops.COPY, Ops.BUFFER, Ops.BUFFER_VIEW,
-                     Ops.CONST, Ops.BIND, Ops.DEVICE, Ops.MSELECT, Ops.MSTACK, Ops.DEFINE_GLOBAL,
-                     Ops.DEFINE_LOCAL, Ops.DEFINE_REG, Ops.LOAD, Ops.KERNEL}
-
 def find_permutes(a:UOp, b:UOp, assign:UOp):
   if not (permutes:=[s for s in b.toposort(gate=lambda s:s.op not in ALWAYS_CONTIGUOUS)
                      if s.op in GroupOp.Movement and s.op not in {Ops.RESHAPE, Ops.EXPAND, Ops.PAD, Ops.SHRINK}]): return
@@ -104,79 +98,9 @@ earliest_rewrites = PatternMatcher([
   (UPat(Ops.CONTIGUOUS, name="root", src=(UPat(Ops.BUFFER),)), lambda root: root.src[0].forced_reshape(root.shape).rtag(root.tag)),
 ])
 
-# *****************
-# 1. add realize where we have to
-
-def realize(ctx:dict[UOp, None], tr:UOp) -> None: ctx[tr] = None
-
-def realize_parents(ctx:dict[UOp, None], rb:UOp) -> None:
-  for s in rb.src:
-    if s.base.op not in ALWAYS_CONTIGUOUS: ctx[s] = None
-
-def realize_assign(ctx:dict[UOp, None], a:UOp) -> None:
-  if a.src[1].op not in ALWAYS_CONTIGUOUS: ctx[a.src[1]] = None
-  # if it's a kernel, we don't realize it
-  if a.src[1].op is not Ops.KERNEL: ctx[a] = None
-
-do_realize = PatternMatcher([
-  # always realize SINK parents
-  (UPat(Ops.SINK, name="s"), lambda ctx,s: ctx.update((x.base, None) for x in s.src if x.base.op not in ALWAYS_CONTIGUOUS)),
-  # always realize ASSIGN/COPY/BUFFER_VIEW/CONTIGUOUS
-  (UPat({Ops.COPY, Ops.BUFFER_VIEW, Ops.CONTIGUOUS}, name="tr"), realize),
-  # realize parents of COPY, MSELECT, MSTACK
-  (UPat((Ops.COPY, Ops.MSELECT, Ops.MSTACK), name="rb"), realize_parents),
-  # realize input to assign (might be optimized out)
-  (UPat(Ops.ASSIGN, name="a"), realize_assign),
-])
-
-class WrappedContig:
-  def __init__(self, x): self.x = x
-  def __repr__(self): return f"C({self.x})"
-add_contiguous = PatternMatcher([(UPat(GroupOp.All, name="x"), lambda ctx,x: x.replace(tag=WrappedContig(x.tag)).realize() if x in ctx else None),])
-remove_contig_tags = PatternMatcher([(UPat(GroupOp.All, name="x"), lambda x: x.replace(tag=x.tag.x) if isinstance(x.tag, WrappedContig) else None)])
-
-# *****************
-# 2. mark all children
-
-@dataclass
-class ChildrenContext: children: dict[UOp, list[UOp]]|None = None
-def extract_children(ctx:ChildrenContext, x:UOp):
-  if ctx.children is not None: return
-  children_map = x.get_consumer_map()
-  ctx.children = {}
-  for k,v in children_map.items():
-    # NOTE: we treat mstack children like sink here
-    non_sink_children = [u for u in v if u.op not in {Ops.SINK, Ops.MSTACK}]
-    if len(non_sink_children) <= 1: continue
-    # NOTE: this gate shouldn't be here
-    if k.op_in_backward_slice_with_self(Ops.REDUCE_AXIS) and k.op_in_backward_slice_with_self(Ops.BUFFER, Ops.CONTIGUOUS):
-      ctx.children[k] = non_sink_children
-
-def mark_children(ctx:ChildrenContext, x:UOp):
-  assert ctx.children is not None
-  new_srcs = [(UOp(Ops.CHILD, s.dtype, src=(UOp(Ops.CHILDREN, s.dtype, (s,), arg=len(ctx.children[s])),),
-                   arg=(ctx.children[s].index(x), len(ctx.children[s]))) if s in ctx.children else s) for s in x.src]
-  return x.replace(src=tuple(new_srcs))
-
-pm_children = PatternMatcher([
-  (UPat(Ops.SINK, name="x"), extract_children),
-  (UPat(GroupOp.All-{Ops.CHILD, Ops.CHILDREN, Ops.SINK}, name="x"), mark_children),
-])
-
 # *****************
 # 3a. rangeify (movement)
-
-@dataclass
-class RangeifyContext:
-  # block on parent until all children have been seen
-  seen_children: dict[UOp, dict[int, UOp]] = field(default_factory=dict)
-  seen_child: dict[UOp, Any] = field(default_factory=dict)
-  progress: int = 0
-
-  # create ranges
-  range_idx: Iterator[int] = field(default_factory=itertools.count)
-  def new_range(self, s:sint, axistype:AxisType=AxisType.LOOP):
-    return UOp.range(s, next(self.range_idx), axistype) if resolve(s!=1) else UOp.const(dtypes.index, 0)
+# NOTE: this can be deleted after the cleanup is refactored
 
 def map_reshape(idx:UOp, r:UOp):
   acc = 1
@@ -241,150 +165,6 @@ pm_mops = PatternMatcher([
   (UPat(Ops.PAD, name="r").f(Ops.INDEX, allow_any_len=True, name="idx"), map_pad),
 ])
 
-# *****************
-# 3b. rangeify (ops)
-
-# bufferization can happen in three ways
-#   1. there's an explicit REALIZE in the graph
-#   2. the ranges from the children don't match and we have to create a buffer (only on children)
-#   3. might_end_axis triggers because we should be closing a loop to save compute
-
-def map_partial_realize(ctx:RangeifyContext, x:UOp, idx:UOp):
-  if x.arg is None: return None  # map_contiguous can handle this
-  # NOTE: all partial contiguous can safely be replaced by full contiguous. we should be able to match old functionality like this
-  if not (RANGEIFY > 1): return idx.replace(src=(x.replace(arg=None),)+idx.src[1:])
-  ranges = []
-  new_ranges = []
-  passthrough_idx = []
-  for i,s in enumerate(x.shape):
-    if i not in x.arg:
-      ranges.append(idx.src[1+i])
-      continue
-    passthrough_idx.append(idx.src[1+i])
-    ranges.append(ctx.new_range(s))
-    new_ranges.append(ranges[-1])
-  # TODO: this should be able to be global or local
-  ret = x.src[0].index(*ranges).bufferize(*[x for x in new_ranges if x.op is not Ops.CONST],
-                                          arg=BufferizeOpts(device=None, addrspace=AddrSpace.LOCAL))
-  return ret.index(*passthrough_idx)
-
-def map_realize(ctx:RangeifyContext, x:UOp):
-  if x.arg is not None: return None
-  ranges = [ctx.new_range(s) for s in x.shape]
-  return x.src[0].index(*ranges).bufferize(*x.src[1:], *ranges, arg=BufferizeOpts(device=x.device), tag=x.src[0].tag)
-
-def map_reduce(ctx:RangeifyContext, idx:UOp, red:UOp):
-  rngs = list(idx.src[1:])
-  new_ranges = []
-  for i,s in enumerate(red.src[0].shape):
-    if i in red.arg[1]:
-      rngs[i] = ctx.new_range(s, axistype=AxisType.REDUCE)
-      new_ranges.append(rngs[i])
-  return UOp(Ops.REDUCE, red.dtype, src=(red.src[0].index(*rngs),)+tuple(new_ranges), arg=red.arg[0], tag=red.tag)
-
-def index_child(ctx:RangeifyContext, c:UOp, x:UOp, idx:UOp):
-  if c not in ctx.seen_children: ctx.seen_children[c] = {}
-  ctx.seen_children[c][x.arg[0]] = idx
-  # wait here until we have seen all the children
-  if len(ctx.seen_children[c]) != x.arg[1]:
-    ctx.progress += 1
-    if ctx.progress > 10000: raise RuntimeError("children not making progress")
-    raise RewriteNotReady
-  ctx.progress = 0
-
-  if c not in ctx.seen_child:
-    all_rngs = list(zip(*[ch.src[1:] for ch in ctx.seen_children[c].values()]))
-    out_rngs = []
-    end_ranges = []
-    idx_ranges = []
-    # NOTE: locals aren't working, so we only fully bufferize here (unless RANGEIFY > 1)
-    rngs_valids = []
-    for valid_rngs in all_rngs:
-      rngs, valids = zip(*[(r.get_idx(), r.get_valid()) for r in valid_rngs])
-      # if a range has a 1 src, it's the same as UOp.const(dtypes.index, 0)
-      same_rngs = [x if x.op is not Ops.RANGE or resolve(x.src[0] != 1) else UOp.const(dtypes.index, 0) for x in rngs]
-      rngs_valids.append((rngs, valids, all_same(same_rngs)))
-    all_all_same = all(same_rngs for _,_,same_rngs in rngs_valids)
-    for i,(rngs,valids,same_rngs) in enumerate(rngs_valids):
-      # we compare the ranges without their valids
-      if same_rngs and (all_all_same or RANGEIFY > 1):
-        # the new valid is the OR of all the children valids
-        minimum_valid = functools.reduce(operator.or_, valids, UOp.const(dtypes.bool, False))
-        out_rngs.append(minimum_valid.where(rngs[0], UOp.invalid()).simplify())
-      else:
-        out_rngs.append(ctx.new_range(c.shape[i]))
-        end_ranges.append(out_rngs[-1])
-        idx_ranges.append(i)
-    ctx.seen_child[c] = (out_rngs, idx_ranges, end_ranges)
-  else:
-    out_rngs, idx_ranges, end_ranges = ctx.seen_child[c]
-    for i,nr in zip(idx_ranges, end_ranges): out_rngs[i] = nr
-  # index based on the shared ranges
-  ret = c.index(*out_rngs)
-  # if all ranges aren't the same between children, we have to bufferize
-  if len(idx_ranges) > 0:
-    if len(idx_ranges) == len(out_rngs):
-      # this is a global bufferize
-      ret = ret.bufferize(*end_ranges, arg=BufferizeOpts(device=x.device))
-    else:
-      assert RANGEIFY > 1, "this isn't supported with RANGEIFY=1"
-      ret = ret.bufferize(*end_ranges, arg=BufferizeOpts(device=None, addrspace=AddrSpace.LOCAL))
-    ret = ret.index(*[idx.src[1+i] for i in idx_ranges])
-  return ret
-
-def children_gate(ctx:RangeifyContext, idx:UOp, c:UOp):
-  if len(ctx.seen_children[c]) != c.arg: raise RuntimeError("all children should have been seen by now")
-  return idx.replace(src=(idx.src[0].src[0],)+idx.src[1:])
-
-def might_end_axis(idx:UOp):
-  if idx.arg is None: return None
-  # TODO: write a proper cost function here
-  if not idx.op_in_backward_slice_with_self(Ops.BUFFER, Ops.REALIZE, Ops.BUFFERIZE): return None
-  if not idx.op_in_backward_slice_with_self(Ops.REDUCE_AXIS): return None
-  to_end_axis = []
-  for i,a in enumerate(idx.src[1:]):
-    # in RANGEIFY=1, always realize
-    if not (RANGEIFY > 1) or any(x.arg > idx.arg for x in a.toposort() if x.op is Ops.RANGE):
-      to_end_axis.append(i)
-  if to_end_axis: return idx.replace(src=(idx.src[0].realize(arg=tuple(to_end_axis)),)+idx.src[1:], arg=None)
-  return idx.replace(arg=None)
-
-def unprocessed_index(x:UOp): raise RuntimeError(f"unprocessed index on {x.src[0].op}")
-
-pm_rangeify = pm_mops+PatternMatcher([
-  # sink contigs to kick it off
-  (UPat(Ops.REALIZE, src=(UPat(),), name="x", allow_any_len=True), map_realize),
-  # if there's an INDEX it can support partial contig
-  (UPat(Ops.INDEX, src=(UPat(Ops.REALIZE, src=(UPat(),), name="x"),), allow_any_len=True, name="idx"), map_partial_realize),
-
-  # if there are new ended children, tag the SINK
-  (UPat(Ops.INDEX, src=(UPat(Ops.CHILD, src=(UPat(name="c"), ), name="x"),), allow_any_len=True, name="idx"), index_child),
-  (UPat(Ops.INDEX, src=(UPat(Ops.CHILDREN, name="c"),), allow_any_len=True, name="idx"), children_gate),
-
-  # if we come across this, remove it. it was a CHILD unused in an INDEX
-  (UPat(Ops.CHILD, src=(UPat(Ops.CHILDREN, src=(UPat.var("x"),)),)), lambda x: x),
-
-  # CONST (or DEFINE_VAR) can't have axes. remove INDEX when we get here
-  (UPat(Ops.INDEX, src=(UPat((Ops.CONST, Ops.DEFINE_VAR), name="c"),)), lambda c: c.replace(src=())),
-
-  # handle arg on any op with weight. old endrange stuff
-  (UPat(Ops.INDEX, src=(UPat(GroupOp.Elementwise.union({Ops.REDUCE_AXIS})),), allow_any_len=True, name="idx"), might_end_axis),
-
-  # handle assign
-  (UPat(Ops.INDEX, src=(UPat(Ops.ASSIGN, name="assign"),), allow_any_len=True, name="x"),
-    lambda x,assign: assign.replace(src=tuple([s.index(*x.src[1:]) for s in assign.src])+(assign.src[0],)) \
-        if assign.src[1].op is not Ops.KERNEL else None),
-
-  # move MAP through elementwise ALU / reduce. these are the items with cost
-  (UPat(Ops.INDEX, src=(UPat(GroupOp.Elementwise.union(
-    {Ops.STORE, Ops.COPY, Ops.BUFFER_VIEW, Ops.DEVICE, Ops.BIND, Ops.CONTIGUOUS, Ops.NOOP})),), allow_any_len=True, name="x"),
-   lambda x: x.src[0].replace(src=tuple([s.index(*x.src[1:]) for s in x.src[0].src]))),
-  (UPat(Ops.INDEX, src=(UPat(Ops.REDUCE_AXIS, name="red"),), allow_any_len=True, name="idx"), map_reduce),
-
-  # assert if there's any index we didn't process
-  (UPat(GroupOp.All-{Ops.REALIZE, Ops.BUFFERIZE, Ops.MSELECT, Ops.MSTACK}).f(Ops.INDEX, name="x"), unprocessed_index),
-])
-
 # *****************
 # 3.5 cleanups
 
@@ -500,7 +280,7 @@ to_bufferview = PatternMatcher([
 ])
 
 DEVICE_MAX_BUFS = {"METAL": 31, "WEBGPU": 8} # TODO: get from device?
-def limit_bufs(ctx:RangeifyContext, root:UOp):
+def limit_bufs(ctx:IndexingContext, root:UOp):
   if (device:=root._device) is None: return None # no device, index related calculations
   device = device if isinstance(device, str) else device[0].split(":")[0]
   if not (MAX_BUFS:=getenv("MAX_KERNEL_BUFFERS", DEVICE_MAX_BUFS.get(device, 0))): return None
@@ -760,19 +540,9 @@ def get_rangeify_map(sink:UOp) -> dict[UOp, UOp]:
   tsink = graph_rewrite(sink, add_tags, ctx=uop_list, bottom_up=True, name="number the uops")
 
   tsink = graph_rewrite(tsink, earliest_rewrites+replace_contiguous, ctx={}, name="earliest rewrites")
-  realize_map: dict[UOp, None] = {}
-  graph_rewrite(tsink, do_realize, ctx=realize_map, name="Input Graph")
 
-  FAST = getenv("FAST", 1)
-  if FAST:
-    rctx: RangeifyContext|IndexingContext
-    tsink, rctx = run_rangeify(tsink, realize_map, FAST > 1)
-  else:
-    # NOTE: we don't use contiguous here, contiguous is a user op
-    tsink = graph_rewrite(tsink, add_contiguous, ctx=realize_map, bottom_up=True, name="add realize")
-    tsink = graph_rewrite(tsink, remove_contig_tags, name="remove contiguous tags")
-    tsink = graph_rewrite(tsink, pm_children, ctx=ChildrenContext(), bottom_up=True, name="get children")
-    tsink = graph_rewrite(tsink, pm_rangeify, ctx=(rctx:=RangeifyContext()), bottom_up=True, name="rangeify")
+  # convert movement ops to ranges
+  tsink, rctx = run_rangeify(tsink, getenv("DEBUG_RANGEIFY", 0))
 
   # NOTE: sym (vs symbolic_simple) breaks things here because ranges with len 1 aren't handled right
   tsink = graph_rewrite(tsink, symbolic_simple+pm_reduce_unparented, name="symbolic")  # this supports const folding

tinygrad/uop/__init__.py

@@ -12,9 +12,6 @@ class Ops(FastEnum):
   NOOP = auto(); SINK = auto(); UNIQUE = auto(); DEVICE = auto(); KERNEL = auto(); PRECAST = auto(); REWRITE_ERROR = auto()  # noqa: E702
   SENTINEL = auto()
 
-  # track children
-  CHILD = auto(); CHILDREN = auto() # noqa: E702
-
   # buffer ops
   COPY = auto(); BUFFER = auto(); BUFFER_VIEW = auto(); MSELECT = auto(); MSTACK = auto() # noqa: E702
 
@@ -24,7 +21,6 @@ class Ops(FastEnum):
 
   # ops that adjust the behavior of the scheduler
   CONTIGUOUS = auto(); CONTIGUOUS_BACKWARD = auto(); DETACH = auto(); FUSE = auto() # noqa: E702
-  REALIZE = auto()
 
   # blocks in linearizer (only used there)
   BLOCK = auto(); BLOCKSTART = auto(); BLOCKEND = auto(); BLOCKFINAL = auto() # noqa: E702

tinygrad/uop/ops.py

@@ -367,7 +367,6 @@ class UOp(MathTrait, metaclass=UOpMetaClass):
     return self.src[0] if self.op is Ops.WHERE and self.src[2].arg is Invalid else UOp.const(dtypes.bool, self.arg is not Invalid)
   def reduce(self, *src:UOp, **kwargs): return UOp(Ops.REDUCE, kwargs.pop('dtype', self.dtype), src=(self,)+src, **kwargs)
   def contiguous(self, *args, **kwargs): return UOp(Ops.CONTIGUOUS, dtype=self.dtype, src=(self,)+args, **kwargs)
-  def realize(self, *args, **kwargs): return UOp(Ops.REALIZE, dtype=self.dtype, src=(self,)+args, **kwargs)
   def contiguous_backward(self): return self.alu(Ops.CONTIGUOUS_BACKWARD)
   def bufferize(self, *args, **kwargs): return UOp(Ops.BUFFERIZE, dtype=self.dtype, src=(self,)+args, **kwargs)
   def fuse(self): return self.alu(Ops.FUSE)
@@ -954,8 +953,8 @@ class TrackedPatternMatcher(PatternMatcher):
         continue
       match_stats[p][1] += 1
       try: ret = match(uop, ctx)
-      except Exception as e:
-        if TRACK_MATCH_STATS >= 2 and active_rewrites and not isinstance(e, RewriteNotReady):
+      except Exception:
+        if TRACK_MATCH_STATS >= 2 and active_rewrites:
           active_rewrites[-1].matches.append((track_uop(uop), track_uop(UOp(Ops.REWRITE_ERROR, src=uop.src, arg=str(sys.exc_info()[1]))), p.location))
         raise
       if ret is not None and ret is not uop:
@@ -998,7 +997,6 @@ if TRACK_MATCH_STATS or PROFILE:
 # *** simple graph rewrite engine ***
 
 with Context(SPEC=0): SENTINEL = UOp(Ops.SENTINEL)
-class RewriteNotReady(Exception): pass
 class BottomUpGate(Exception): pass
 class RewriteContext:
   def __init__(self, pm, bpm, ctx=None):
@@ -1038,10 +1036,6 @@ class RewriteContext:
               if test_n in seen: raise RuntimeError("infinite loop in fixed_point_rewrite")
               seen.add(test_n)
               new_n, test_n = test_n, self.cached_bpm_rewrite(test_n)
-          except RewriteNotReady:
-            # try the full thing again later
-            stack.appendleft((n, 0, n))
-            continue
           except BottomUpGate:
             # if the bpm matching raised a gate, we are done with this node and dont continue down the srcs
             self.replace[n] = new_n
@@ -1055,7 +1049,7 @@ class RewriteContext:
         tmp = []
         for x in new_n.src:
           if (rx:=self.replace.get(x, SENTINEL)) is SENTINEL:
-            # if some new sources aren't ready, we try this again later
+            # if some new sources aren't ready, we try this again later. happens with on_stack, maybe should remove?
             stack.appendleft((n, 1, new_n))
             break
           tmp.append(rx)

tinygrad/uop/spec.py

@@ -239,11 +239,6 @@ full_spec = PatternMatcher([
   # where on index in rhs position is fine
   (UPat(Ops.WHERE, src=(UPat(dtype=dtypes.bool), UPat(), UPat(dtype=dtypes.index))), lambda: True),
 
-  # all children is fine
-  (UPat(Ops.CHILDREN), lambda: True),
-  # child must have CHILDREN parent
-  (UPat(Ops.CHILD, src=(UPat(Ops.CHILDREN),)), lambda: True),
-
   # all rewrite error are okay
   (UPat(Ops.REWRITE_ERROR), lambda: True),
 
@@ -251,8 +246,6 @@ full_spec = PatternMatcher([
   (UPat(Ops.BUFFER_VIEW, src=(UPat((Ops.INDEX, Ops.LOAD)),)), lambda: True),
   # bufferize (must be on ranges)
   (UPat(Ops.BUFFERIZE, src=(UPat(),), allow_any_len=True, name="x"), lambda x: all(y.op in {Ops.RANGE, Ops.CONST} for y in x.src[1:])),
-  # realize with one src is fine
-  (UPat(Ops.REALIZE, src=(UPat(),)), lambda: True),
   # intermediate index
   (UPat(Ops.INDEX, src=(UPat(),), allow_any_len=True, name="x"), lambda x: all(y.dtype == dtypes.index for y in x.src[1:]) or None),
   (UPat(Ops.REDUCE, src=(UPat(),), allow_any_len=True, name="x"), lambda x: all(y.dtype == dtypes.index for y in x.src[1:])),

tinygrad/viz/serve.py

@@ -19,9 +19,8 @@ uops_colors = {Ops.LOAD: "#ffc0c0", Ops.STORE: "#87CEEB", Ops.CONST: "#e0e0e0",
                Ops.INDEX: "#e8ffa0", Ops.WMMA: "#efefc0", Ops.MULTI: "#f6ccff", Ops.KERNEL: "#3e7f55",
                **{x:"#D8F9E4" for x in GroupOp.Movement}, **{x:"#ffffc0" for x in GroupOp.ALU}, Ops.THREEFRY:"#ffff80", Ops.BUFFER_VIEW: "#E5EAFF",
                Ops.BLOCK: "#C4A484", Ops.BLOCKEND: "#C4A4A4", Ops.BUFFER: "#B0BDFF", Ops.COPY: "#a040a0", Ops.FUSE: "#FFa500",
-               Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D", Ops.REALIZE: "#C1C14D",
-               Ops.CHILDREN: "#80ffc0", Ops.CHILD: "#80fff0", Ops.BUFFERIZE: "#FF991C", Ops.REWRITE_ERROR: "#ff2e2e",
-               Ops.SUBSTITUTE: "#ffff00"}
+               Ops.ALLREDUCE: "#ff40a0", Ops.MSELECT: "#d040a0", Ops.MSTACK: "#d040a0", Ops.CONTIGUOUS: "#FFC14D",
+               Ops.BUFFERIZE: "#FF991C", Ops.REWRITE_ERROR: "#ff2e2e", Ops.SUBSTITUTE: "#ffff00"}
 
 # VIZ API