diff --git a/tinygrad/codegen/linearize.py b/tinygrad/codegen/linearize.py index 27fc665d02..86001c1149 100644 --- a/tinygrad/codegen/linearize.py +++ b/tinygrad/codegen/linearize.py @@ -122,7 +122,7 @@ def block_merge(ctx, x:UOp): pm_block_merge = PatternMatcher([ (UPat((Ops.BLOCKEND, Ops.BLOCK), name="x"), block_merge), # double BLOCKFORK multiplies the forking (like if there's 3 forks into 2 forks, that's 6 total forks) - (UPat(Ops.BLOCKFORK, name="f", src=(UPat(Ops.BLOCKFORK, name="f2"))), lambda f,f2: f.replace(src=f2.src, arg=f.arg*f2.arg)), + (UPat(Ops.BLOCKFORK, name="f", src=(UPat(Ops.BLOCKFORK, name="f2"),)), lambda f,f2: f.replace(src=f2.src, arg=f.arg*f2.arg)), ]) def block_finalize(block:UOp): diff --git a/tinygrad/codegen/symbolic.py b/tinygrad/codegen/symbolic.py index 654c8410f7..bd0ab69c67 100644 --- a/tinygrad/codegen/symbolic.py +++ b/tinygrad/codegen/symbolic.py @@ -49,14 +49,17 @@ symbolic_simple = PatternMatcher([ (UPat.var("x") * 0, lambda x: x.const_like(float("nan") if isinstance(x.arg, float) and (math.isnan(x.arg) or math.isinf(x.arg)) else 0)), # ** constant folding ** # TODO: add const folding for Ops.THREEFRY - (UPat(GroupOp.ALU-{Ops.THREEFRY}, name="a", src=UPat((Ops.VCONST, Ops.CONST))), - lambda a: a.const_like(exec_alu(a.op, a.dtype, [x.arg for x in a.src], False))), + (UPat(GroupOp.Unary, src=(UPat((Ops.VCONST, Ops.CONST)),), name="a"), lambda a: a.const_like(exec_alu(a.op, a.dtype, [a.src[0].arg], False))), + (UPat(GroupOp.Binary-{Ops.THREEFRY}, src=(UPat((Ops.VCONST, Ops.CONST)),)*2, name="a"), + lambda a: a.const_like(exec_alu(a.op, a.dtype, [a.src[0].arg, a.src[1].arg], False))), + (UPat(GroupOp.Ternary, src=(UPat((Ops.VCONST, Ops.CONST)),)*3, name="a"), + lambda a: a.const_like(exec_alu(a.op, a.dtype, [a.src[0].arg, a.src[1].arg, a.src[2].arg], False))), # bool MUL is AND, ADD/MAX is OR. prevents other rules to rewrite bool ADD/MUL incorrectly (UPat.var('x', dtype=dtypes.bool) * UPat.var('y', dtype=dtypes.bool), lambda x,y: x&y), (UPat.var('x', dtype=dtypes.bool) + UPat.var('y', dtype=dtypes.bool), lambda x,y: x|y), (UPat.var('x', dtype=dtypes.bool).maximum(UPat.var('y', dtype=dtypes.bool)), lambda x,y: x|y), # *** cast/bitcast *** - (UPat(Ops.CAST, name="root", src=UPat.cvar("c")), lambda root, c: root.const_like(c.arg)), + (UPat(Ops.CAST, name="root", src=(UPat.cvar("c"),)), lambda root, c: root.const_like(c.arg)), (UPat((Ops.CAST, Ops.BITCAST), name="root"), lambda root: root.src[0] if root.dtype == root.src[0].dtype else None), (UPat(Ops.BITCAST, name="root", src=(UPat.cvar("c"),)), fold_bitcast), # ** pow ** diff --git a/tinygrad/engine/grouper.py b/tinygrad/engine/grouper.py index 50c86fd3f0..78f82835a7 100644 --- a/tinygrad/engine/grouper.py +++ b/tinygrad/engine/grouper.py @@ -103,7 +103,7 @@ reorder_view = PatternMatcher([ (UPat(Ops.COPY, src=(UPat(), UPat(Ops.VIEW, name="v")), name="copy"), lambda copy,v: v.contiguous().copy_to_device(copy.device) \ if prod(v.shape) < prod(v.base.shape) else v.base.copy_to_device(copy.device, clone=copy.arg).view(v.st)), # put UnaryOps before EXPANDs - (UPat(GroupOp.Unary, src=UPat(Ops.VIEW, src=(UPat.var("inp"),), name="v"), name="alu"), + (UPat(GroupOp.Unary, src=(UPat(Ops.VIEW, src=(UPat.var("inp"),), name="v"),), name="alu"), lambda inp,v,alu: inp.alu(alu.op).view(v.st) if resolve(prod(alu.shape) > v.st.real_size()) else None), # put CAST after expanding BUFFER (UPat(Ops.VIEW, src=(UPat(Ops.CAST, src=(UPat.var("x"),)),), name="v"), lambda x,v: x.view(x.st+v.st).cast(v.dtype) if getenv("CAST_AFTER_EXPAND") diff --git a/tinygrad/ops.py b/tinygrad/ops.py index 5beeaf4c76..e95e15de7a 100644 --- a/tinygrad/ops.py +++ b/tinygrad/ops.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Any, Optional, Union, Callable, cast, TYPE_CHECKING, Type, get_args +from typing import Any, Optional, Union, Callable, cast, TYPE_CHECKING, Type, get_args, Sequence import sys, time, functools, itertools, math, operator, hashlib, os, types, pickle, pathlib, inspect, weakref from enum import auto, IntEnum, Enum from dataclasses import dataclass, field @@ -811,27 +811,31 @@ def deconstruct_function(fxn:Callable) -> tuple: ret = fxn.__code__, new_globals, fxn.__name__, fxn.__defaults__ return pickle.loads(pickle.dumps(ret)) if getenv("TEST_PICKLE") else ret -def get_universal_match(p:UPat, fxn:Callable): - if 'ctx' in inspect.signature(fxn).parameters: +@functools.cache +def upat_interpret(p:UPat, fxn:Callable) -> Callable: + real_fxn = types.FunctionType(*deconstruct_function(fxn)) + if 'ctx' in inspect.signature(real_fxn).parameters: def universal_match(uop, ctx): for match in p.match(uop, {}): - if (ret:=fxn(ctx=ctx, **match)) is not None: return ret + if (ret:=real_fxn(ctx=ctx, **match)) is not None: return ret # pylint: disable=not-callable + return None else: - def universal_match(uop, ctx): + def universal_match(uop, _): for match in p.match(uop, {}): - if (ret:=fxn(**match)) is not None: return ret + if (ret:=real_fxn(**match)) is not None: return ret # pylint: disable=not-callable + return None return universal_match class PatternMatcher: - def __init__(self, patterns:list[tuple[UPat, Callable]]): - self.patterns = patterns + def __init__(self, patterns:Sequence[tuple[UPat, Callable|tuple]]): + # if this comes from a pickle, we reconstruct the lambda functions here + self.patterns:list[tuple[UPat, Callable]] = [(p,types.FunctionType(*fxn) if isinstance(fxn, tuple) else fxn) for p,fxn in patterns] # NOTE: use of DefaultDict here is very dangerous! all keys will live for the lifetime of the PatternMatcher! self.pdict: dict[Ops, list[tuple[UPat, Callable, set]]] = {} # uop is required, arg is optional for p,fxn in self.patterns: assert p.op is not None - tuple_fxn = fxn if isinstance(fxn, tuple) else deconstruct_function(fxn) - match = get_universal_match(p, types.FunctionType(*tuple_fxn)) + match = upat_interpret(p, fxn) for uop in p.op: self.pdict.setdefault(uop, []).append((p, match, p.early_reject)) def __reduce__(self): return PatternMatcher, ([(x,deconstruct_function(fxn) if fxn.__name__ == "" else fxn) for x,fxn in self.patterns],) diff --git a/tinygrad/renderer/cstyle.py b/tinygrad/renderer/cstyle.py index 238d1a86d5..b9ca112931 100644 --- a/tinygrad/renderer/cstyle.py +++ b/tinygrad/renderer/cstyle.py @@ -261,8 +261,8 @@ class IntelRenderer(OpenCLRenderer): opts=("l0","l0","l0","u1","u1","u1"), swizzle=(((4,5,6),(0,1,2,3,7,8,9)), ((0,1,2),(7,8,9,3,4,5,6))))] string_rewrite = PatternMatcher([ - (UPat(Ops.CAST, dtype=dtypes.bfloat16, src=(UPat.var('x', dtype=dtypes.float))), lambda ctx,x: f"intel_convert_bfloat16_as_ushort({ctx[x]})"), - (UPat(Ops.CAST, dtype=dtypes.float, src=(UPat.var('x', dtype=dtypes.bfloat16))), lambda ctx,x: f"intel_convert_as_bfloat16_float({ctx[x]})"), + (UPat(Ops.CAST, dtype=dtypes.bfloat16, src=(UPat.var('x', dtype=dtypes.float),)), lambda ctx,x: f"intel_convert_bfloat16_as_ushort({ctx[x]})"), + (UPat(Ops.CAST, dtype=dtypes.float, src=(UPat.var('x', dtype=dtypes.bfloat16),)), lambda ctx,x: f"intel_convert_as_bfloat16_float({ctx[x]})"), ]) + OpenCLRenderer.string_rewrite def render_kernel(self, function_name, kernel, bufs, uops, prefix=None) -> str: @@ -456,12 +456,15 @@ class AMDRenderer(CStyleLanguage): (UPat(GroupOp.ALU, dtypes.bool, name="alu", src=(UPat.var("x", dtype=dtypes.bfloat16), UPat.var("y", dtype=dtypes.bfloat16))), lambda alu,x,y: UOp(alu.op, dtypes.bool, (x.cast(dtypes.float), y.cast(dtypes.float)), alu.arg)), # add float intermediate casting for bfloat16 - (UPat(Ops.CAST, name="x", src=UPat.var("y", dtypes.bfloat16)),lambda x,y: y.cast(dtypes.float).cast(x.dtype) if x.dtype!=dtypes.float else None), - (UPat(Ops.CAST, dtypes.bfloat16, UPat.var("x")),lambda x: x.cast(dtypes.float).cast(dtypes.bfloat16) if x.dtype!=dtypes.float else None), + (UPat(Ops.CAST, name="x", src=(UPat.var("y", dtypes.bfloat16),)), + lambda x,y: y.cast(dtypes.float).cast(x.dtype) if x.dtype!=dtypes.float else None), + (UPat(Ops.CAST, dtypes.bfloat16, (UPat.var("x"),)), + lambda x: x.cast(dtypes.float).cast(dtypes.bfloat16) if x.dtype!=dtypes.float else None), # bfloat16 casting (UPat.cvar('x', dtypes.bfloat16), lambda x: cast_float_to_bf16(UOp.const(dtypes.float, x.arg))), - (UPat(Ops.CAST, dtypes.float, UPat.var("x", dtypes.bfloat16)), lambda x: (x.bitcast(dtypes.ushort).cast(dtypes.uint)<<16).bitcast(dtypes.float)), - (UPat(Ops.CAST, dtype=dtypes.bfloat16, src=UPat.var("x", dtype=dtypes.float)), cast_float_to_bf16)]) + extra_pm + (UPat(Ops.CAST, dtypes.float, (UPat.var("x", dtypes.bfloat16),)), + lambda x: (x.bitcast(dtypes.ushort).cast(dtypes.uint)<<16).bitcast(dtypes.float)), + (UPat(Ops.CAST, dtype=dtypes.bfloat16, src=(UPat.var("x", dtype=dtypes.float),)), cast_float_to_bf16)]) + extra_pm def render_vector_prefix(self, dtype:DType) -> str: vec, scal = self.render_dtype(dtype), self.render_dtype(dtype.scalar())