mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-08-30 18:26:08 +00:00
hcq2: use shrink.bitcast (#17653)
* hcq2: shrink.bitcast
* x
* x
* x
* s
* x
* Dx
* Revert "hotfix: disable HCQ2"
This reverts commit a57188ea6d.
* x
This commit is contained in:
nimlgen
@@ -94,7 +94,7 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: "0"
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: "0"
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -190,7 +190,7 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: "0"
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -233,7 +233,7 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: "0"
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
@@ -279,7 +279,7 @@ jobs:
|
||||
shell: bash -e -o pipefail {0}
|
||||
env:
|
||||
DEV: ${{ matrix.dev }}
|
||||
HCQ2: "0"
|
||||
HCQ2: ${{ matrix.dev == 'AMD' && '1' || '0' }}
|
||||
if: github.repository_owner == 'tinygrad'
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
|
||||
@@ -57,6 +57,35 @@ def _test_uops_result(output_dtype, uops, res):
|
||||
run_uops([out], [buf])
|
||||
return np.frombuffer(buf.as_memoryview(), _to_np_dtype(output_dtype))[0]
|
||||
|
||||
@unittest.skipUnless(isinstance(Device[Device.DEFAULT].renderer, CStyleLanguage) and
|
||||
dtypes.uint64 in Device[Device.DEFAULT].renderer.supported_dtypes(), "requires C-style pointer bitcast and 64-bit ints")
|
||||
class TestBitcastBufferView(unittest.TestCase):
|
||||
@Context(SPEC=2)
|
||||
def test_render(self):
|
||||
buf = UOp.param(0, dtypes.uint32, (4,))
|
||||
uops = to_uops_list([buf.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0).store(1)], ren=Device[Device.DEFAULT].renderer)
|
||||
idx = next(u for u in uops if u.op is Ops.INDEX and u.src[0].op is Ops.BITCAST)
|
||||
self.assertEqual(idx.src[0].src[0].op, Ops.SHRINK)
|
||||
Device[Device.DEFAULT].renderer.render(uops)
|
||||
|
||||
@Context(SPEC=2)
|
||||
def test_load(self):
|
||||
val = 0x1122334455667788
|
||||
src, out = UOp.param(0, dtypes.uint32, (4,)), UOp.param(1, dtypes.uint64, (1,))
|
||||
ibuf = Buffer(Device.DEFAULT, 4, dtypes.uint32, initial_value=np.array([0, 0x55667788, 0x11223344, 0], dtype=np.uint32).tobytes())
|
||||
obuf = Buffer(Device.DEFAULT, 1, dtypes.uint64).allocate()
|
||||
run_uops([out.index(0).store(src.shrink(((1, 3),)).bitcast(dtypes.uint64).index(0))], [ibuf, obuf])
|
||||
self.assertEqual(np.frombuffer(obuf.as_memoryview(), dtype=np.uint64)[0], val)
|
||||
|
||||
@Context(SPEC=2)
|
||||
def test_store(self):
|
||||
val = 0x1122334455667788
|
||||
dst = UOp.param(0, dtypes.uint32, (6,))
|
||||
buf = Buffer(Device.DEFAULT, 6, dtypes.uint32, initial_value=bytes(24))
|
||||
view = dst.shrink(((1, 5),)).bitcast(dtypes.uint64) # two stores through one view: it must inline, not get a declared vector-pointer
|
||||
run_uops([view.index(0).store(val ^ 0xff), view.index(1).store(val)], [buf])
|
||||
self.assertEqual(np.frombuffer(buf.as_memoryview(), dtype=np.uint64, count=2, offset=4).tolist(), [val ^ 0xff, val])
|
||||
|
||||
class TestUOps(unittest.TestCase):
|
||||
def _equal(self, v1, v2):
|
||||
assert isinstance(v2, (float, int, bool))
|
||||
|
||||
@@ -233,10 +233,11 @@ pm_reduce_local = pm_wmma_add+PatternMatcher([
|
||||
(UPat(Ops.SINK, name="sink"), merge_reduce_ends),
|
||||
])+pm_clean_up_group_sink
|
||||
|
||||
def is_shape_changing_bitcast(u:UOp): return u.op is Ops.BITCAST and u.shape != u.src[0].shape
|
||||
def maybe_load(u:UOp): return u.load() if u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL, AddrSpace.REG) else u
|
||||
pm_add_loads = PatternMatcher([
|
||||
# BITCAST?
|
||||
(UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"), lambda x: x.replace(src=tuple([maybe_load(u) for u in x.src]))),
|
||||
(UPat(GroupOp.Elementwise|{Ops.REDUCE,Ops.WMMA,Ops.STACK}, name="x"),
|
||||
lambda x: None if is_shape_changing_bitcast(x) else x.replace(src=tuple(map(maybe_load, x.src)))),
|
||||
(UPat(Ops.STORE, name="x"), lambda x: x.replace(src=(x.src[0], maybe_load(x.src[1]))+x.src[2:])),
|
||||
])
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ base_rewrite = PatternMatcher([
|
||||
(UPat(Ops.CAST, name="x"), lambda ctx,x: f"__builtin_convertvector({ctx[x.src[0]]}, {ctx.render_type(x)})" \
|
||||
if x.max_numel() > 1 and x.addrspace is AddrSpace.REG else None),
|
||||
(UPat(Ops.CAST, name="x"), lambda ctx,x: f"({ctx.render_cast(x, ctx[x.src[0]])})"),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: ctx[x.src[0]] if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"(({ctx._render_dtype(x.dtype, addrspace=x.addrspace)})({ctx[x.src[0]]}))"
|
||||
if x.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"__builtin_bit_cast({ctx.render_type(x)}, ({ctx.render_type(x.src[0])})({ctx[x.src[0]]}))"),
|
||||
|
||||
# GPU stuff
|
||||
@@ -237,7 +238,7 @@ class CStyleLanguage(Renderer):
|
||||
if (u.op is not Ops.CAST or u.max_numel() == 1) and ((u.op is Ops.CAST and u.src[0].op is Ops.CONST) or \
|
||||
u.op in {Ops.INDEX, Ops.SHRINK, Ops.CUSTOMI} or \
|
||||
(u.op is Ops.LOAD and u.src[0].addrspace == AddrSpace.REG and child_count[u] == 1) or \
|
||||
(u.op is Ops.CAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \
|
||||
(u.op in {Ops.CAST, Ops.BITCAST} and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)) or \
|
||||
(u.op in {Ops.STACK, *(GroupOp.ALU-{Ops.WHERE}), Ops.CAST, Ops.BITCAST} and child_count[u] == 1 and not getenv("EXPAND_SSA"))):
|
||||
r[u] = l
|
||||
else:
|
||||
@@ -318,7 +319,8 @@ class OpenCLRenderer(CStyleLanguage):
|
||||
extra_matcher = create_non_native_float_pats((dtypes.bfloat16,)) + pm_manual_bf16_cast
|
||||
|
||||
string_rewrite = PatternMatcher([
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_{ctx.render_dtype(x.dtype)}(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"
|
||||
if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None),
|
||||
# bfloat16 constants need to be rendered as their bit pattern since bf16 is stored as ushort
|
||||
(UPat.cvar("c").cast(dtypes.bfloat16), lambda ctx,c: f"{(struct.unpack('I', struct.pack('f', float_to_bf16(c.val)))[0] >> 16)}u"),
|
||||
# load/store image (OpenCL)
|
||||
@@ -369,7 +371,8 @@ class MetalRenderer(CStyleLanguage):
|
||||
]) + pm_manual_bf16_cast
|
||||
|
||||
string_rewrite = PatternMatcher([
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"as_type<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"
|
||||
if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None),
|
||||
]) + base_rewrite
|
||||
|
||||
def render_kernel(self, function_name, kernel, bufs, uops, prefix=None):
|
||||
@@ -425,7 +428,8 @@ class CUDARenderer(CStyleLanguage):
|
||||
(UPat(Ops.CAST, dtypes.fp8s, UPat.var("x", dtypes.fp8s), name='y'), lambda x,y: x.cast(dtypes.float).cast(y.dtype) if x.dtype!=y.dtype else None),
|
||||
])
|
||||
string_rewrite = PatternMatcher([
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"),
|
||||
(UPat(Ops.BITCAST, name="x"), lambda ctx,x: f"tg_bitcast<{ctx.render_dtype(x.dtype)}>(({ctx.render_dtype(x.src[0].dtype)})({ctx[x.src[0]]}))"
|
||||
if x.addrspace not in (AddrSpace.GLOBAL, AddrSpace.LOCAL) else None),
|
||||
]) + base_rewrite
|
||||
|
||||
def render_vector_prefix(self, dt:DType, count:int) -> str:
|
||||
|
||||
@@ -89,7 +89,7 @@ class PythonProgram(Program['PythonDevice']):
|
||||
if g: _store(m, o+j, v, src_dtypes[1])
|
||||
i += 1
|
||||
continue
|
||||
if u.op is Ops.AFTER: values[u] = src_values[0]
|
||||
if u.op is Ops.AFTER or (u.op is Ops.BITCAST and u.addrspace in (AddrSpace.GLOBAL, AddrSpace.LOCAL)): values[u] = src_values[0]
|
||||
elif u.op is Ops.PARAM and u.addrspace is AddrSpace.ALU: values[u] = [pvals.pop(0)] * warp_size
|
||||
elif u.op in {Ops.PARAM, Ops.BUFFER}:
|
||||
storage_fmt = storage_fmt_for_dtype(u.dtype)
|
||||
@@ -114,7 +114,8 @@ class PythonProgram(Program['PythonDevice']):
|
||||
if ox < 0 or ox >= u.src[0]._shape[1] or oy < 0 or oy >= u.src[0]._shape[0]: ret.append((m, None))
|
||||
else: ret.append((m, ox*4 + oy*u.src[0]._shape[1]*4))
|
||||
else:
|
||||
for m,o in zip(src_values[0], src_values[1]): ret.append((m,o))
|
||||
scale = u.src[0].dtype.itemsize // u.src[0].src[0].dtype.itemsize if u.src[0].op is Ops.BITCAST else 1
|
||||
for m,o in zip(src_values[0], src_values[1]): ret.append((m[0], m[1]+o*scale) if isinstance(m, tuple) else (m, o*scale))
|
||||
values[u] = ret
|
||||
elif u.op is Ops.RANGE:
|
||||
if u not in values: values[u] = [0] * warp_size
|
||||
|
||||
@@ -8,7 +8,7 @@ from tinygrad.device import Device, Buffer, BufferSpec, Compiled, LRUAllocator,
|
||||
from tinygrad.device import ProfileDeviceEvent, ProfileGraphEntry, ProfileGraphEvent
|
||||
from tinygrad.uop.ops import Ops, sint, UOp, UPat, PatternMatcher, KernelInfo, graph_rewrite, rewrite_group, GroupOp
|
||||
from tinygrad.uop.symbolic import symbolic
|
||||
from tinygrad.dtype import dtypes, truncate
|
||||
from tinygrad.dtype import dtypes, truncate, DType
|
||||
from tinygrad.runtime.support.hcq import MMIOInterface, HCQBuffer
|
||||
from tinygrad.runtime.support.memory import BumpAllocator
|
||||
from tinygrad.renderer import Renderer, Estimates
|
||||
@@ -40,6 +40,9 @@ def unwrap_mstack(u):
|
||||
if u.op is Ops.MSTACK: return tuple(x for s in u.src for x in unwrap_mstack(s))
|
||||
return unwrap_mstack(u.src[0]) if u.op is Ops.MSELECT else (u,)
|
||||
|
||||
def unwrap_view(v:UOp) -> tuple[UOp, int]:
|
||||
return unwrap_view(v.src[0]) if v.op is Ops.BITCAST else (v.src[0], v.src[1].val) if v.op is Ops.SHRINK else (v, 0)
|
||||
|
||||
def is_value_known_at_link(val:UOp) -> bool:
|
||||
runtime_reads = [u for u in val.toposort() if u.op in (Ops.LOAD, Ops.INDEX)]
|
||||
addressed_bufs = [b for g in val.toposort() if g.op is Ops.GETADDR for b in unwrap_mstack(g.buf_uop)]
|
||||
@@ -48,16 +51,17 @@ def is_value_known_at_link(val:UOp) -> bool:
|
||||
return not val.variables() and not runtime_reads and all(b.op is not Ops.PARAM or b.tag is not None for b in addressed_bufs)
|
||||
|
||||
def make_patches(buf:UOp, patches:Sequence[tuple[sint, UOp]]) -> tuple[UOp, ...]:
|
||||
def _mk_store(ps:list[tuple[sint, UOp]], tag:str|None) -> UOp:
|
||||
offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const(off // buf.dtype.itemsize, dtypes.int) for off,_ in ps))
|
||||
vals = UOp(Ops.STACK, ps[0][1].dtype, tuple(val for _,val in ps))
|
||||
return buf.index(offs, dtype=vals.dtype).store(vals).rtag(tag)
|
||||
groups:dict[tuple[str|None, DType, sint], list[tuple[sint, UOp]]] = collections.defaultdict(list)
|
||||
for off, val in patches:
|
||||
tag = "link" if is_value_known_at_link(val) else "inputs" if val.op is Ops.GETADDR else None
|
||||
groups[(tag, (v:=(val.bitcast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val)).dtype, off % v.dtype.itemsize)].append((off, v))
|
||||
|
||||
patches = [(off, val.cast(buf.dtype) if val.dtype.itemsize == buf.dtype.itemsize else val) for off, val in patches]
|
||||
link, runtime = partition(patches, lambda p: is_value_known_at_link(p[1]))
|
||||
inputs, runtime = partition(runtime, lambda p: p[1].op is Ops.GETADDR)
|
||||
return tuple(_mk_store(list(ps), tag) for cls, tag in ((link, "link"), (inputs, "inputs"), (runtime, None))
|
||||
for _, ps in itertools.groupby(sorted(cls, key=lambda p: p[1].dtype), key=lambda p: p[1].dtype))
|
||||
ret, bit = [], buf.dtype.itemsize
|
||||
for (tag, dt, r), ps in groups.items():
|
||||
view = buf.shrink(((r // bit, (max(off for off,_ in ps) + dt.itemsize) // bit),)).bitcast(dt)
|
||||
offs = UOp(Ops.STACK, dtypes.int, tuple(UOp.const((off - r) // dt.itemsize, dtypes.int) for off,_ in ps))
|
||||
ret.append(view.index(offs).store(UOp(Ops.STACK, dt, tuple(val for _,val in ps))).rtag(tag))
|
||||
return tuple(ret)
|
||||
|
||||
def make_binary_patch(buf:UOp, blob:bytes) -> UOp:
|
||||
data = UOp(Ops.BINARY, src=(), arg=blob).bitcast(buf.dtype)
|
||||
@@ -328,14 +332,16 @@ def make_addr_table(call:UOp, gaddrs:list[UOp], name:str) -> tuple[UOp, dict[UOp
|
||||
return table, reads, fills, {g:slots[bare[g]] for g in gaddrs}
|
||||
|
||||
def make_gather_loop(patches:list[UOp], table:UOp, slots:dict[UOp, int], lt_patches:list[UOp]) -> dict[UOp, UOp]:
|
||||
(dst,), words = dedup(p.buf_uop for p in patches), [(off.val, slots[val]) for p in patches for off, val in zip(p.src[0].src[1].src, p.src[1].src)]
|
||||
(dst,), words = dedup(p.buf_uop for p in patches), [(unwrap_view(p.src[0].src[0])[1] + off.val*(val.dtype.itemsize//p.buf_uop.dtype.itemsize),
|
||||
slots[val]) for p in patches for off,val in zip(p.src[0].src[1].src, p.src[1].src)]
|
||||
|
||||
# build a runtime loop that writes every input address
|
||||
pairs = UOp.placeholder((2*len(words),), dtypes.uint32, next(UOp.unique_num), device=dst.device).rtag("systems")
|
||||
lt_patches.append(make_binary_patch(pairs, struct.pack(f'<{2*len(words)}I', *itertools.chain(*words))))
|
||||
r = UOp.range(len(words), next(UOp.unique_num), dtype=dtypes.int, src=(pairs, dst))
|
||||
off, slot = ((pairs.index(2*r+i).load() % bound).cast(dtypes.int) for i, bound in ((0, dst.max_numel()-1), (1, table.max_numel())))
|
||||
return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: dst.index(off, dtype=table.dtype).store(table.index(slot).load()).end(r)}
|
||||
patch = dst.shrink(((off, off+table.dtype.itemsize//dst.dtype.itemsize),)).bitcast(table.dtype).index(0).store(table.index(slot).load()).end(r)
|
||||
return {p: UOp(Ops.NOOP) for p in patches} | {patches[0]: patch}
|
||||
|
||||
def is_input_addr(g:UOp) -> bool: return all(x.op is Ops.PARAM and x.tag is None for x in unwrap_mstack(g.buf_uop))
|
||||
|
||||
@@ -391,14 +397,13 @@ pm_replace_params = PatternMatcher([
|
||||
|
||||
def resolve_getaddr_view(bv:UOp, g:UOp) -> UOp:
|
||||
base = bv.src[0].after(*g.src[0].src[1:] if g.src[0].op is Ops.AFTER else ())
|
||||
if bv.op is Ops.BITCAST: return UOp(Ops.GETADDR, src=(base,), arg=g.arg)
|
||||
itemsize = bv.src[0].dtype.itemsize if bv.src[0].without_after.op in (Ops.BUFFER, Ops.MSTACK, Ops.MSELECT) else bv.dtype.itemsize
|
||||
return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64)
|
||||
addr = UOp(Ops.GETADDR, src=(base,), arg=g.arg)
|
||||
return addr if bv.op is Ops.BITCAST else addr + UOp.const(bv.src[1].val * bv.dtype.itemsize, dtypes.uint64)
|
||||
|
||||
pm_early_simplify = PatternMatcher([
|
||||
(UPat(Ops.GETADDR, src=(UPat((Ops.SHRINK, Ops.BITCAST), name="bv").or_after(),), name="g"), resolve_getaddr_view),
|
||||
(UPat(Ops.INDEX, src=(UPat(Ops.SHRINK, name="bv"),), allow_any_len=True, name="x"),
|
||||
lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))),
|
||||
(UPat(Ops.SHRINK, src=(UPat(Ops.SHRINK, name="bv"), UPat(), UPat()), name="x"),
|
||||
lambda bv,x: bv.src[0].shrink(((start:=bv.src[1]+x.src[1], start+x.src[2]),))),
|
||||
])
|
||||
|
||||
# *****************
|
||||
@@ -507,11 +512,13 @@ def fold_binary(buf:UOp, blob:UOp) -> UOp:
|
||||
b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[:len(blob.arg)] = blob.arg
|
||||
return UOp(Ops.NOOP)
|
||||
|
||||
def fold_const_store(buf:UOp, off:UOp, val:UOp) -> UOp:
|
||||
def fold_const_store(view:UOp, off:UOp, val:UOp) -> UOp:
|
||||
buf, start = unwrap_view(view)
|
||||
for off,val in zip(off.src, val.src):
|
||||
for b,v in zip((bs:=mb.bufs if isinstance((mb:=buf.buffer), MultiBuffer) else (mb,)), val.src if val.op is Ops.STACK else (val,)*len(bs)):
|
||||
data = struct.pack(f'<{v.dtype.fmt}', truncate[v.dtype]((v.src[0] if v.op is Ops.CAST else v).val))
|
||||
b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[(bo:=off.val*buf.dtype.itemsize):bo+len(data)] = data
|
||||
bo = start*buf.dtype.itemsize + off.val*val.dtype.itemsize
|
||||
b.ensure_allocated().as_memoryview(force_zero_copy=True, no_sync=True).cast('B')[bo:bo+len(data)] = data
|
||||
return UOp(Ops.NOOP)
|
||||
|
||||
def resolve_getaddr(buf:UOp, g:UOp) -> UOp:
|
||||
@@ -532,10 +539,10 @@ pm_resolve_patches = PatternMatcher([
|
||||
(UPat(Ops.GETADDR, src=(UPat(name="buf"),), name="g"), resolve_getaddr),
|
||||
|
||||
# folders
|
||||
(UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True)
|
||||
.store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast()).index(UPat(Ops.RANGE), allow_any_len=True).load())
|
||||
.end(UPat(Ops.RANGE)), fold_binary),
|
||||
(UPat({Ops.BUFFER, Ops.MSTACK}, name="buf").index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store),
|
||||
(UPat(name="buf").index(UPat(Ops.RANGE), allow_any_len=True).store(UPat.any(UPat(Ops.BINARY, name="blob"), UPat(Ops.BINARY, name="blob").bitcast())
|
||||
.index(UPat(Ops.RANGE), allow_any_len=True).load()).end(UPat(Ops.RANGE)), fold_binary),
|
||||
(UPat((Ops.BITCAST, Ops.SHRINK, Ops.BUFFER, Ops.MSTACK), name="view")
|
||||
.index(UPat(Ops.STACK, name="off")).store(UPat(Ops.STACK, name="val")), fold_const_store),
|
||||
])
|
||||
|
||||
pm_assert_no_afters = PatternMatcher([(UPat(Ops.AFTER, name="a"), lambda a: panic(RuntimeError, f"AFTER left at hcq_link: {a.src[0].op}"))])
|
||||
|
||||
Reference in New Issue
Block a user