diff --git a/tinygrad/codegen/__init__.py b/tinygrad/codegen/__init__.py index ab880c1fda..826f6139fb 100644 --- a/tinygrad/codegen/__init__.py +++ b/tinygrad/codegen/__init__.py @@ -16,7 +16,7 @@ from tinygrad.codegen.expander import migrate_indexing, expander from tinygrad.codegen.devectorizer import load_store_folding, load_store_indexing, devectorize, pm_reduce, \ ReduceContext, correct_load_store, pm_render from tinygrad.codegen.linearize import block_create, pm_blockend_merge, block_merge, pm_finalize, BlockContext -from tinygrad.codegen.opt import pm_optimize +from tinygrad.codegen.opt import pm_get_optimization, pm_do_optimize from tinygrad.codegen.opt.swizzler import view_left, view_right, fix_kernel_ops @dataclass @@ -55,7 +55,8 @@ def _get_rewrites_for_renderer(opts:Renderer, linearizer:bool, _QUANTIZE, _DEVEC ret.extend(rewrites_for_views) # this is kernel.py - ret.append(RewriteStep(pm_optimize, ctx=lambda _: opts, name="optimize ast")) + ret.append(RewriteStep(pm_get_optimization, ctx=lambda _: opts, name="get optimization")) + ret.append(RewriteStep(pm_do_optimize, ctx=lambda _: opts, name="optimize ast")) if _QUANTIZE and opts.device in {"CPU", "DSP"}: ret.append(RewriteStep(pm_quant, name="quantize")) ret.append(RewriteStep(pm_lowerer, get_index, name="lowerer", bottom_up=True)) diff --git a/tinygrad/codegen/linearize.py b/tinygrad/codegen/linearize.py index 07706383b9..54ff2cbfc7 100644 --- a/tinygrad/codegen/linearize.py +++ b/tinygrad/codegen/linearize.py @@ -3,7 +3,7 @@ import heapq from collections import defaultdict from dataclasses import dataclass, replace from tinygrad.uop.ops import UOp, Ops, PatternMatcher, UPat, GroupOp -from tinygrad.helpers import dedup, all_same, flatten, getenv +from tinygrad.helpers import dedup, all_same, flatten, BLOCK_REORDER # NOTE: any toposort should be valid here, unlike last time this isn't required, it's just for speed def block_reorder(lst:list[UOp]) -> list[UOp]: @@ -150,7 +150,7 @@ def make_block_bottom_up(ctx:BlockContext, x:UOp): srcs.append(add_blockends(base_block, new_ctx, current_ctx)) lst = lst[::-1] - if getenv("BLOCK_REORDER", 1): lst = block_reorder(lst) + if BLOCK_REORDER: lst = block_reorder(lst) bb = BasicBlock(tuple(lst), ctx=current_ctx, cnt=child_count, child_ctx=child_ctx) return UOp(Ops.BLOCK, src=tuple(srcs), arg=bb) diff --git a/tinygrad/codegen/opt/__init__.py b/tinygrad/codegen/opt/__init__.py index 202585f8b6..0c507c5681 100644 --- a/tinygrad/codegen/opt/__init__.py +++ b/tinygrad/codegen/opt/__init__.py @@ -2,7 +2,7 @@ from tinygrad.codegen.opt.kernel import Kernel from tinygrad.codegen.opt.heuristic import hand_coded_optimizations -from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops +from tinygrad.uop.ops import UOp, PatternMatcher, UPat, Ops, KernelInfo from tinygrad.helpers import NOOPT, BEAM, USE_TC, getenv from tinygrad.renderer import Renderer from tinygrad.uop.spec import type_verify @@ -19,20 +19,28 @@ def get_optimized_ast(ast:UOp, renderer:Renderer) -> UOp: The Ops.SINK rooted AST transformed to apply the opts and with a KernelInfo in the arg. """ + assert ast.arg is None, "no opt if there's an arg" k = Kernel(ast, opts=renderer) - if ast.arg is not None and ast.arg.opts_to_apply is not None: k.apply_opts(ast.arg.opts_to_apply) - elif not NOOPT: + if not NOOPT: if not k.apply_tensor_cores(USE_TC.value): k.apply_opts(hand_coded_optimizations(k)) if BEAM >= 1: from tinygrad.codegen.opt.search import beam_search, bufs_from_lin kb = Kernel(ast, opts=renderer) rawbufs = bufs_from_lin(kb, allocate=False) k = beam_search(kb, rawbufs, BEAM.value, bool(getenv("BEAM_ESTIMATE", 1))) + return ast.replace(arg=KernelInfo(opts_to_apply=tuple(k.applied_opts))) + +pm_get_optimization = PatternMatcher([ + (UPat(Ops.SINK, name="ast"), lambda ctx,ast: get_optimized_ast(ast, ctx) if ast.arg is None and ast.src[0].st is not None else None), +]) + +def apply_opt(ast:UOp, renderer:Renderer): + k = Kernel(ast, opts=renderer) + k.apply_opts(ast.arg.opts_to_apply) ret = k.get_optimized_ast() if __debug__: type_verify(list(ret.toposort())) return ret -pm_optimize = PatternMatcher([ - (UPat(Ops.SINK, name="ast"), lambda ctx,ast: - get_optimized_ast(ast, ctx) if (ast.arg is None or ast.arg.opts_to_apply is not None) and ast.src[0].st is not None else None), +pm_do_optimize = PatternMatcher([ + (UPat(Ops.SINK, name="ast"), lambda ctx,ast: apply_opt(ast, ctx) if ast.arg is not None and ast.arg.opts_to_apply is not None else None), ]) diff --git a/tinygrad/codegen/opt/heuristic.py b/tinygrad/codegen/opt/heuristic.py index 3fa0ffd5e9..40b3a9d3cc 100644 --- a/tinygrad/codegen/opt/heuristic.py +++ b/tinygrad/codegen/opt/heuristic.py @@ -28,7 +28,7 @@ def hand_coded_optimizations(k:Kernel) -> list[Opt]: return k.applied_opts # are we grouping? (requires local shape support) - if resolve(prod(k.sts[0].shape[i] for i in k.upcastable_dims) <= 2048, False): + if resolve(prod(k.output_shape[i] for i in k.upcastable_dims) <= 2048, False): for sz in [16]: try: k.apply_opt(Opt(OptOps.GROUPTOP, 0, sz)) @@ -62,7 +62,7 @@ def hand_coded_optimizations(k:Kernel) -> list[Opt]: # potentially do more upcasts of non reduce axes based on a heuristic is_dsp = k.opts is not None and k.opts.device == "DSP" upcasted_axis: set[int] = set() - while resolve(prod(k.sts[0].shape[i] for i in k.upcastable_dims) >= 1024): + while resolve(prod(k.output_shape[i] for i in k.upcastable_dims) >= 1024): xb_choices = [] # consider all upcastable axes with 3 or 4 upcast (128 on the DSP) for axis, upcast_amount in itertools.product(k.upcastable_dims, ([128] if not len(upcasted_axis) else []) if is_dsp else [3,4]): diff --git a/tinygrad/helpers.py b/tinygrad/helpers.py index 6ba80fa52a..00865bad7a 100644 --- a/tinygrad/helpers.py +++ b/tinygrad/helpers.py @@ -135,7 +135,7 @@ FUSE_ARANGE, FUSE_CONV_BW = ContextVar("FUSE_ARANGE", 1), ContextVar("FUSE_CONV_ SPLIT_REDUCEOP, NO_MEMORY_PLANNER, RING = ContextVar("SPLIT_REDUCEOP", 1), ContextVar("NO_MEMORY_PLANNER", 0), ContextVar("RING", 1) PICKLE_BUFFERS, PROFILE, LRU = ContextVar("PICKLE_BUFFERS", 1), ContextVar("PROFILE", getenv("VIZ")), ContextVar("LRU", 1) CACHELEVEL, IGNORE_BEAM_CACHE, DEVECTORIZE = ContextVar("CACHELEVEL", 2), ContextVar("IGNORE_BEAM_CACHE", 0), ContextVar("DEVECTORIZE", 1) -DISABLE_COMPILER_CACHE = ContextVar("DISABLE_COMPILER_CACHE", 0) +DISABLE_COMPILER_CACHE, BLOCK_REORDER = ContextVar("DISABLE_COMPILER_CACHE", 0), ContextVar("BLOCK_REORDER", 1) DONT_REALIZE_EXPAND, DONT_GROUP_REDUCES = ContextVar("DONT_REALIZE_EXPAND", 0), ContextVar("DONT_GROUP_REDUCES", 0) QUANTIZE, VALIDATE_WITH_CPU, DISABLE_FAST_IDIV = ContextVar("QUANTIZE", 0), ContextVar("VALIDATE_WITH_CPU", 0), ContextVar("DISABLE_FAST_IDIV", 0) CORRECT_DIVMOD_FOLDING, FUSE_OPTIM = ContextVar("CORRECT_DIVMOD_FOLDING", 0), ContextVar("FUSE_OPTIM", 0)