diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd53bd57ac..41d8831807 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -521,10 +521,7 @@ jobs: - name: Run HCQ2 tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/test_tiny.py - name: Run HCQ2 multi-device tests - run: | - HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_multitensor.py \ - TestMultiTensor.test_simple_add TestMultiTensor.test_shard_reduce \ - TestMultiTensor.test_backward_sum TestMultiTensor.test_matmul_shard_0_0 + run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python -m pytest -n=auto test/backend/test_multitensor.py - name: Run HCQ2 JIT tests run: HCQ_RUNTIME_DEV=PYTHON HCQ2=1 DEV=MOCKKFD+AMD FORWARD_ONLY=1 PYTHONPATH=. python test/unit/test_jit.py - name: Run HCQ2 unit tests @@ -591,7 +588,7 @@ jobs: if: ${{ matrix.backend == 'amd' && matrix.arch == 'gfx950' }} run: PYTHONPATH=. DEV=NULL:HIP:gfx950 MXFP4=1 LLAMA_LAYERS=2 BENCHMARK=3 NULL_ALLOW_COPYOUT=1 NO_HIPCC=1 ROCM_PATH=/opt/rocm JITBEAM=0 examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/profile.sh - name: Run pytest (amd) - run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM test/opt/test_tensor_cores.py --durations=20 + run: python -m pytest -n=auto test/backend/test_ops.py test/backend/test_dtype.py test/backend/test_dtype_alu.py test/backend/test_linearizer.py test/backend/test_randomness.py test/backend/test_jit.py test/backend/test_graph.py test/backend/test_multitensor.py test/device/test_hcq.py test/external/external_test_am.py test/backend/test_asm_gemm.py::TestAsmGEMM --durations=20 - name: Run disk copy tests run: python -m pytest test/unit/test_disk_tensor.py -k test_copy_from_disk - name: Run TRANSCENDENTAL math diff --git a/extra/hcq2/ops_amd2.py b/extra/hcq2/ops_amd2.py index fbdef9f444..6d69671d81 100644 --- a/extra/hcq2/ops_amd2.py +++ b/extra/hcq2/ops_amd2.py @@ -297,6 +297,8 @@ class AMDAllocator(HCQAllocator['AMDDevice']): def _do_map(self, buf:HCQ2Buffer): return self.dev.iface.map(buf._base if buf._base is not None else buf) + def _do_unmap(self, buf:HCQ2Buffer): self.dev.iface.unmap(buf) + @dataclass class AMDQueueDesc: ring: Buffer; read_ptr: Buffer; write_ptr: Buffer; doorbell: Buffer; put_value: Buffer # noqa: E702 @@ -388,15 +390,24 @@ class KFDIface: return hcqbuf def free(self, mem): + self._unmap(mem) + if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) + kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def unmap(self, mem): + self._unmap(mem) + if getattr(mem, '_owns_kfd_handle', False): kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) + + def _unmap(self, mem): gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(gpus), n_devices=1) assert stm.n_success == 1 - if mem.owner == self.dev: - if mem.va_addr: FileIOInterface.munmap(mem.va_addr, mem.size) - kfd.AMDKFD_IOC_FREE_MEMORY_OF_GPU(self.kfd, handle=mem.meta.handle) def map(self, mem): - if mem.owner is not None and mem.owner._is_cpu(): return self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + if mem.owner is not None and mem.owner._is_cpu(): + mapped = self.alloc(mem.size, host=True, cpu_addr=mem.va_addr) + mapped._owns_kfd_handle = True + return mapped c_gpus = (ctypes.c_int32 * 1)(self.gpu_id) stm = kfd.AMDKFD_IOC_MAP_MEMORY_TO_GPU(self.kfd, handle=mem.meta.handle, device_ids_array_ptr=ctypes.addressof(c_gpus), n_devices=1) @@ -468,6 +479,7 @@ class PCIIface(PCIIfaceBase): def require_profile_mode(self): return True def is_wgp_active(self, xcc, se, sa, wgp) -> bool: return True # TODO: account for WGP disablement on some asics. + def unmap(self, mem): self.free(mem) def _compute_props(self): self.ip_versions = self.dev_impl.ip_ver diff --git a/test/helpers.py b/test/helpers.py index ea900d505c..21febed6bc 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -48,6 +48,8 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te else: assert isinstance(t, UOp), f"can't schedule {t}" linear, var_vals = Tensor(t).linear_with_vars() + # test compiling the linear + compile_linear(linear) kernel_cnt = sum((len(call.device) if isinstance(call.device, tuple) else 1) for call in linear.src if call.src[0].op is Ops.SINK or not filter_sink) if kernel_cnt != allowed: @@ -57,8 +59,6 @@ def check_schedule(t:Tensor|list[Tensor]|UOp, allowed:int, to_prerealize:list[Te print("kernel", i+1) print(call.src[0]) raise KernelCountException(allowed, kernel_cnt) - # test compiling the linear - compile_linear(linear) return linear, var_vals def assert_kernel_count(expected:int): diff --git a/tinygrad/codegen/opt/tc.py b/tinygrad/codegen/opt/tc.py index 52df9ce560..6d13624a1d 100644 --- a/tinygrad/codegen/opt/tc.py +++ b/tinygrad/codegen/opt/tc.py @@ -1,6 +1,7 @@ import math, functools from dataclasses import dataclass from tinygrad.dtype import DType, dtypes +from tinygrad.uop.ops import PatternMatcher, UOp, UPat, Ops @dataclass(frozen=True) class TensorCore: # D = A * B + C, A is (M x K), B is (K x N), C and D are (M x N) @@ -135,6 +136,42 @@ amd_cdna4 = amd_cdna_1616128 + amd_cdna_161632 + amd_cdna_161616 def get_amd(arch): return {"gfx942": amd_cdna3, "gfx950": amd_cdna4, "gfx1200": amd_rdna4, "gfx1201": amd_rdna4}.get(arch, amd_rdna3) +pm_validate_wmma_rdna3 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( + src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) + if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) + for j in range(x.max_numel()*2)))), + arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) + for i in range(x.max_numel()))) if x.max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x"), lambda x: x.replace( + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), +]) + +pm_validate_wmma_rdna4 = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( + dtype=dtypes.uint16, + src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) + .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) +]) + +pm_validate_wmma_cdna = PatternMatcher([ + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) + if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), + (UPat(Ops.WMMA, name="x", dtype=dtypes.float), + lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) + if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), +]) # ***** Apple Metal ***** metal = [TensorCore(dims=(8,8,8), threads=32, elements_per_thread=(2,2,2), dtype_in=di, dtype_out=do, diff --git a/tinygrad/engine/realize.py b/tinygrad/engine/realize.py index 92006e87d6..e91e83ea75 100644 --- a/tinygrad/engine/realize.py +++ b/tinygrad/engine/realize.py @@ -140,7 +140,7 @@ class ExecContext: cache: bool = True def _resolve(b:UOp, inputs:tuple[UOp, ...]) -> UOp: - if b.op in (Ops.SLICE, Ops.MSELECT) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) + if b.op in (Ops.SLICE, Ops.MSELECT, Ops.SHRINK) and b.src[0].op is Ops.PARAM: return b.replace(src=(inputs[b.src[0].arg.slot], *b.src[1:])) if b.op is Ops.MSTACK: return b.replace(src=tuple(_resolve(x, inputs) for x in b.src)) return inputs[b.arg.slot] if b.op is Ops.PARAM else b def resolve_params(call:UOp, inputs:tuple[UOp, ...]) -> list[UOp]: return [_resolve(b, inputs) for b in get_call_arg_uops(call)] diff --git a/tinygrad/renderer/llvmir.py b/tinygrad/renderer/llvmir.py index 99da4d1345..05f6e72033 100644 --- a/tinygrad/renderer/llvmir.py +++ b/tinygrad/renderer/llvmir.py @@ -279,43 +279,9 @@ exit: %packed = phi i32 [%packed_bf8, %do_bf8], [%packed_fp8, %do_fp8]\n %trunc (UPat(Ops.WMMA, name="wmma"), lambda ctx, wmma, rdna4=AMDLLVMRenderer.is_rdna4(target.arch), cdna=self.is_cdna: render_wmma_amd(ctx, wmma, cdna, rdna4)) ]) - if self.is_cdna: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.arg[0][2] == 128 and x.src[0].dtype.itemsize <= 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 4 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint64), x.src[1].bitcast(dtypes.uint64), x.src[2])) - if x.max_numel() == 4 and x.src[0].dtype in dtypes.fp8_ocp and x.src[0].max_numel() == 8 else None), - ]) - if target.arch in {"gfx1100", "gfx1151"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.int32), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint32), x.src[1].bitcast(dtypes.uint32), x.src[2])) - if x.src[0].dtype == dtypes.int8 and x.src[0].max_numel() == 16 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.half), lambda x: UOp(Ops.STACK, src=tuple(x.replace( - src=(x.src[0], x.src[1], UOp(Ops.STACK, src=tuple(x.src[2].index(UOp.const(j//2, dtypes.int16)) - if j%2 == 0 else UOp.const(0.0, x.src[2].dtype) - for j in range(x.max_numel()*2)))), - arg=(*x.arg[:4], None)).index(UOp.const(i*2, dtypes.int16)) - for i in range(x.max_numel()))) if x.max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x"), lambda x: x.replace( - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 16 else None), - ]) - if target.arch in {"gfx1200", "gfx1201"}: - self.extra_matcher += PatternMatcher([ - (UPat(Ops.WMMA, name="x", dtype=dtypes.bfloat16), lambda x: x.replace( - dtype=dtypes.uint16, - src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2].bitcast(dtypes.uint16))) - .bitcast(dtypes.bfloat16) if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None), - (UPat(Ops.WMMA, name="x", dtype=dtypes.float), - lambda x: x.replace(src=(x.src[0].bitcast(dtypes.uint16), x.src[1].bitcast(dtypes.uint16), x.src[2])) - if x.max_numel() == 8 and x.src[0].dtype == dtypes.bfloat16 and x.src[0].max_numel() == 8 else None) - ]) + if self.is_cdna: self.extra_matcher += tc.pm_validate_wmma_cdna + if target.arch in {"gfx1100", "gfx1151"}: self.extra_matcher += tc.pm_validate_wmma_rdna3 + if target.arch in {"gfx1200", "gfx1201"}: self.extra_matcher += tc.pm_validate_wmma_rdna4 def supported_dtypes(self): return {d for d in super().supported_dtypes() if (d not in dtypes.fp8_ocp or self.target.arch == "gfx950") and d not in dtypes.fp8_fnuz} diff --git a/tinygrad/runtime/graph/metal.py b/tinygrad/runtime/graph/metal.py index 409cfb973d..8ad6152c4f 100644 --- a/tinygrad/runtime/graph/metal.py +++ b/tinygrad/runtime/graph/metal.py @@ -113,5 +113,5 @@ class MetalGraph(GraphRunner): @staticmethod def supports_uop(batch_devs, new_call:UOp) -> bool: # Metal ICB replay encodes offsets as uint32; reject if any Metal buffer offset exceeds 32-bit range. - if any(b.op is Ops.SLICE and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False + if any(b.op in {Ops.SLICE, Ops.SHRINK} and b.src[1].val * b.src[0].dtype.itemsize > 0xFFFFFFFF for b in new_call.src[1:]): return False return GraphRunner.supports_uop(batch_devs, new_call) diff --git a/tinygrad/runtime/support/hcq2.py b/tinygrad/runtime/support/hcq2.py index 9b0372d035..dd045c5e19 100644 --- a/tinygrad/runtime/support/hcq2.py +++ b/tinygrad/runtime/support/hcq2.py @@ -141,14 +141,14 @@ def _build_wait_cmds(slots:dict[str, int], dep_lanes:list[tuple[tuple, int, int] # opt2: keep latest dep per (dep device, queue, cur lane) latest = {((dep[0][dlane], dep[1]), lane): (dep, dlane) for dep, dlane, lane in sorted(dep_lanes, key=lambda x: x[0][2])} - deps:dict[tuple, list[int|None]] = collections.defaultdict(lambda: [None]*len(devices)) - for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane] = dlane + deps:dict[tuple, dict[int, list[int]]] = collections.defaultdict(lambda: collections.defaultdict(list)) + for (_, lane), (dep, dlane) in latest.items(): deps[dep][lane].append(dlane) waits = [] - for (ddevs, dqueue, dtag), lanes in deps.items(): - sig = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) - for dl, d in zip(lanes, devices)]) - waits.append(UOp(Ops.INS, arg="wait", src=(sig, UOp.const(dtag + 1, dtypes.uint64)))) + for (ddevs, dqueue, dtag), by_lane in deps.items(): + for ls in itertools.zip_longest(*(by_lane[lane] for lane in range(len(devices)))): + s = UOp.mstack(*[make_signal(d, tag="sentinel_signal") if dl is None else make_signal(ddevs[dl], slots[dqueue]) for dl, d in zip(ls, devices)]) + waits.append(UOp(Ops.INS, arg="wait", src=(s, UOp.const(dtag + 1, dtypes.uint64)))) return waits, {dtag for _, _, dtag in deps} def _build_finalizers(batch:list[tuple[UOp, tuple[str, ...]]], batch_info:list[tuple[tuple[str, ...], str]], @@ -383,7 +383,7 @@ def resolve_getaddr_slice(bv:UOp, g:UOp) -> UOp: return UOp(Ops.GETADDR, src=(base,), arg=g.arg) + UOp.const(bv.src[1].val * itemsize, dtypes.uint64) pm_early_simplify = PatternMatcher([ - (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat(Ops.SLICE, name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), + (UPat(Ops.GETADDR, src=(UPat.any(sl:=UPat((Ops.SLICE, Ops.SHRINK), name="bv"), sl.after(allow_any_len=True)),), name="g"), resolve_getaddr_slice), (UPat(Ops.INDEX, src=(UPat(Ops.SLICE, name="bv"),), allow_any_len=True, name="x"), lambda bv,x: x.replace(src=(bv.src[0], x.src[1] + bv.src[1].cast(x.src[1].dtype), *x.src[2:]))), ]) @@ -623,6 +623,8 @@ class HCQAllocator(LRUAllocator[HCQDeviceType], Generic[HCQDeviceType]): if not hasattr(self, '_do_map'): raise NotImplementedError("map failed: no method implemented") return self._do_map(buf) + def _do_unmap(self, mb): self.dev.iface.free(mb) + @suppress_finalizing def _free(self, buf:HCQ2Buffer, options:BufferSpec|None=None): if options is not None and options.external_ptr is not None: return @@ -631,6 +633,6 @@ class HCQAllocator(LRUAllocator[HCQDeviceType], Generic[HCQDeviceType]): def _unmap(self, mb): self.dev.synchronize() - self.dev.iface.free(mb) + self._do_unmap(mb) def _offset(self, buf, size:int, offset:int) -> HCQ2Buffer: return buf.offset(offset=offset, size=size) diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index 304d1fe1a3..c2b654a7af 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -23,6 +23,7 @@ class AllocCtx: bases: set[UOp] = field(default_factory=set) assigns: list[UOp] = field(default_factory=list) replacements: list[UOp] = field(default_factory=list) + views: set[UOp] = field(default_factory=set) def tag_uop(ctx:AllocCtx, x:UOp): if x.tag is not None: return None @@ -63,40 +64,37 @@ def replace_contig_with_store_after(u:UOp): def replace_store_after_with_contig(u:UOp, src:UOp): assigned_to = u while assigned_to.op in {Ops.BITCAST, Ops.AFTER, Ops.UNSHARD}: assigned_to = assigned_to.src[0].base - if assigned_to.op not in {Ops.BUFFER, Ops.SLICE}: return src.contiguous(tag=u.tag) + if assigned_to.op is not Ops.BUFFER: return src.contiguous(tag=u.tag) def _make_buffer_view(src:UOp) -> UOp|None: - """If movement ops on src collapse to a contiguous range, return SLICE. Otherwise None.""" - if (offset := src.contiguous_view_offset()) is None: return None - buf = src.base - if buf.op is Ops.SLICE: - byte_offset = buf.src[1].val * buf.src[0].dtype.itemsize + offset * src.dtype.itemsize - buf = buf.src[0] - if byte_offset % buf.dtype.itemsize != 0: return None - offset = byte_offset // buf.dtype.itemsize - return UOp(Ops.SLICE, src.dtype, (buf, UOp.const(offset)), src.numel()) + if (cv := src.contiguous_view()) is None: return None + (buf, offset), size = cv, src.max_numel() * src.element_size() // cv[0].element_size() + if buf.op is not Ops.BUFFER: return None + # NB: make offset a UOp.variable here to do the offset computation in the kernels + return buf[offset:offset+size].bitcast(src.dtype) -def contiguous_mops_to_view(c:UOp, src:UOp): - """MOPS(BUFFER) → SLICE when movement ops collapse to a contiguous range.""" +def contiguous_mops_to_view(ctx:AllocCtx, c:UOp, src:UOp): + """MOPS(BUFFER) → SHRINK when movement ops collapse to a contiguous range.""" buf = src.base - if buf.op not in {Ops.BUFFER, Ops.SLICE, Ops.UNSHARD}: return None - if src.op is Ops.RESHAPE and src.src[0].op in {Ops.BUFFER, Ops.SLICE} and c.op is not Ops.BITCAST: return None - if c.op is not Ops.BITCAST and src.op is Ops.BUFFER: return None + while buf.op is Ops.BITCAST: buf = buf.src[0].base + if buf.op not in {Ops.BUFFER, Ops.UNSHARD}: return None # no symbolic shape if not all_int(c.shape): return None if buf.op is not Ops.UNSHARD and (view := _make_buffer_view(src)) is not None: - view = (view.replace(dtype=c.dtype, arg=c.numel()) if c.op is Ops.BITCAST else view).reshape(c.shape) - return c.replace(src=(view,)) if c.op is Ops.COPY else view + ctx.views.add(view) + view = view.reshape(c.shape) + return c.replace(src=(view,)+c.src[1:]) if c.op in {Ops.COPY, Ops.STORE} else view - # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then create SLICE on the resolved result + # for UNSHARD tensors, use multi_pm to resolve per-shard movement ops, then create SHRINK on the resolved result if not isinstance(c.device, str): from tinygrad.schedule.multi import multi_pm resolved = graph_rewrite(src, multi_pm, name="multi_buffer_view") if resolved.op is not Ops.UNSHARD: return None if (view := _make_buffer_view(resolved.src[0])) is None: return None - return view.reshape(resolved.src[0].shape).unshard(resolved.arg, resolved.src[1:]).contiguous(tag=c.tag) + ctx.views.add(view) + return view.reshape(resolved.src[0].shape).unshard(resolved.arg, resolved.src[1:]) return None @@ -151,8 +149,9 @@ pm_early_transform_tensor_graph = PatternMatcher([ # resolve TUPLE+GETTUPLE (for precompiled calls) (UPat(Ops.GETTUPLE, src=(UPat(Ops.TUPLE, name="t"),), name="g"), lambda g,t: t.src[g.arg]), - # fold MOPS+BITCAST over BUFFER/SLICE into SLICE when movement ops collapse to contiguous range - (UPat((Ops.BITCAST, Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BUFFER}, name="src"),), name="c"), contiguous_mops_to_view), + # fold MOPS+BITCAST over BUFFER into SHRINK when movement ops collapse to contiguous range + (UPat((Ops.COPY, Ops.CONTIGUOUS), src=(UPat(GroupOp.Movement|{Ops.BITCAST}, name="src"),), name="c"), contiguous_mops_to_view), + (UPat(Ops.STORE, src=(UPat(Ops.BITCAST, name="src"), UPat()), name="c", allow_any_len=True), contiguous_mops_to_view), # remove contiguous on movement ops before a copy on disk (UPat(GroupOp.Movement-{Ops.SHRINK, Ops.RESHAPE}, name="x").f(Ops.CONTIGUOUS).f(Ops.COPY, name="copy"), lambda x,copy: @@ -201,6 +200,8 @@ def replace_input_buffer(ctx:AllocCtx, b:UOp): return UOp.param(len(ctx.replacements)-1, b.dtype, b.shape, b.device, addrspace=b.addrspace if b.addrspace is not None else AddrSpace.GLOBAL) +def replace_input_view(ctx:AllocCtx, b:UOp): return replace_input_buffer(ctx, b) if b in ctx.views else None + pm_finalize_call = PatternMatcher([ (UPat(Ops.AFTER, name="x"), finalize_after), (UPat(Ops.COPY, name="x"), lambda ctx,x: ctx.assigns.append(x) if isinstance(x.device, str) and x.device.startswith(("DISK", "TINYFS")) else None), @@ -210,8 +211,9 @@ pm_replace_buf = PatternMatcher([ # replace BUFFER with PARAM for cache key normalization (UPat(Ops.BUFFER, src=(UPat(),), name="b"), lambda ctx,b: replace_input_buffer(ctx, b) if isinstance(b.arg, ParamArg) and b.addrspace is AddrSpace.GLOBAL else None), - # replace SLICE with PARAM. this rewrite is bottom up so BUFFERs we don't need won't be in the input - (UPat(Ops.SLICE, src=(UPat(Ops.BUFFER), UPat(Ops.CONST, dtype=dtypes.weakint)), name="b"), replace_input_buffer), + # replace SHRINK with PARAM + (UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), name="b", allow_any_len=True), replace_input_view), + (UPat(Ops.BITCAST, src=(UPat.any(UPat(Ops.SHRINK, src=(UPat(Ops.BUFFER),), allow_any_len=True), UPat(Ops.BUFFER)),), name="b"), replace_input_view), # strip value from BIND for cache key normalization, so different values hit same cache (UPat(Ops.BIND, src=(UPat(Ops.PARAM), UPat(Ops.CONST)), name="b"), replace_input_buffer), ]) @@ -229,7 +231,7 @@ def transform_to_call(big_sink:UOp) -> tuple[UOp, dict[UOp, UOp]]: big_sink = graph_rewrite(big_sink, add_tags, ctx=ctx, bottom_up=True, name="number the uops") # here we can break the tensor graph. this is the only place you need to maintain numbered tags - big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, name="early transform tensor graph") + big_sink = graph_rewrite(big_sink, pm_early_transform_tensor_graph, ctx=ctx, name="early transform tensor graph") # here we construct the final buffer_map: as-built nodes -> their final storage. values are never keys graph_rewrite(big_sink, pm_finalize_call, ctx=ctx, name="finalize call") diff --git a/tinygrad/uop/ops.py b/tinygrad/uop/ops.py index 9ae599b987..28e2a047ba 100644 --- a/tinygrad/uop/ops.py +++ b/tinygrad/uop/ops.py @@ -568,9 +568,9 @@ class UOp(RandMixin, metaclass=UOpMetaClass): in_tuple = self.src[0] if self.op is Ops.FUNCTION else self assert in_tuple.op is Ops.TUPLE, f"gettuple requires FUNCTION or TUPLE source, got {self.op}" return UOp(Ops.GETTUPLE, src=(self,), arg=idx) - def group(*srcs:UOp|None): # pylint: disable=no-self-argument + def group(*srcs:UOp|None, **kwargs): # pylint: disable=no-self-argument if len(srcs) == 1 and isinstance(srcs[0], UOp): return srcs[0] - return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None])) + return UOp(Ops.GROUP, src=tuple([x for x in srcs if x is not None]), **kwargs) def index(self, *srcs:UOp|int|None, **kwargs): new_srcs: list[UOp] = [UOp.const(x) if isinstance(x, int) else x for x in srcs if x is not None] if len(new_srcs) == 1 and new_srcs[0].op is Ops.CONST and self.op is Ops.STACK: return self.src[new_srcs[0].val] @@ -822,7 +822,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): unique_num = itertools.count(0) def getaddr(self, device=None) -> UOp: - if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self + if self.without_after.op not in {Ops.BUFFER, Ops.SLICE, Ops.SHRINK, Ops.BINARY, Ops.MSTACK, Ops.MSELECT, Ops.PARAM}: return self return UOp(Ops.GETADDR, src=(self,), arg=device or to_tuple(self.device)[0]) @staticmethod def new_buffer(device:str|tuple[str, ...], size:int, dtype:DType, num=None): @@ -901,8 +901,7 @@ class UOp(RandMixin, metaclass=UOpMetaClass): while len(s.src) and s.op not in {Ops.BUFFER, Ops.PARAM, Ops.STAGE, Ops.MSTACK}: s = s.src[0] return s - def contiguous_view_offset(self) -> int|None: - """If movement ops on a BUFFER collapse to a contiguous range, return `offset` in elements. Otherwise None.""" + def contiguous_view(self) -> tuple[UOp, int]|None: from tinygrad.schedule.rangeify import pm_mops from tinygrad.uop.symbolic import symbolic @@ -915,7 +914,10 @@ class UOp(RandMixin, metaclass=UOpMetaClass): idx = self.flatten().index(UOp.range(self.numel(), 0)) out = graph_rewrite(idx, pm_mops+symbolic+pm_contiguous_view_offset, ctx=self, name="contiguous_view_offset") - return out.val if out.op is Ops.CONST and isinstance(out.val, int) else None + if out.op is not Ops.INDEX or not (b:=out.src[0]).tag or (c:=out.src[1]).op is not Ops.CONST or not isinstance(c.val, int): return None + return b.rtag(None), c.val + + def contiguous_view_offset(self) -> int|None: return None if (view := self.contiguous_view()) is None else view[1] def has_buffer_identity(self, after_ok=False): """Check if this UOp has a concrete buffer identity in the graph (RESHAPE/UNSHARD -> BUFFER chain).""" @@ -932,18 +934,16 @@ class UOp(RandMixin, metaclass=UOpMetaClass): @property def buffer(self) -> Buffer|MultiBuffer: - if self.op in {Ops.CONTIGUOUS, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer + if self.op in {Ops.CONTIGUOUS, Ops.CONTIGUOUS_BACKWARD, Ops.RESHAPE, Ops.UNSHARD, Ops.DETACH, Ops.AFTER}: return self.src[0].buffer # this buffer can process disk tensors and simple movement ops - if self is not self.base: - buf = self.base.buffer - assert isinstance(buf, Buffer), "must be a Buffer for movement ops" - offset = self.contiguous_view_offset() - if offset is None: raise RuntimeError(f"non-contiguous view is not supported for {buf.device} buffer") - return buf.view(prod(self.max_shape), self.dtype, offset*self.dtype.itemsize) - if self.op is Ops.BITCAST: - buf = self.src[0].buffer - assert isinstance(buf, Buffer), "must be a Buffer for BITCAST" - return buf.view(prod(self.max_shape), self.dtype, 0) + if self is not self.base or self.op is Ops.BITCAST: + if (cv := self.contiguous_view()) is None: raise RuntimeError(f"non-contiguous view is not supported for {self.device} buffer") + buf, offset = (b:=cv[0]).base.buffer, cv[1] + if isinstance(buf, MultiBuffer): + mbuf = MultiBuffer.__new__(MultiBuffer) + mbuf.bufs = [x.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) for x in buf.bufs] + return mbuf + return buf.view(prod(self.max_shape), self.dtype, offset*b.dtype.itemsize) if self.op is Ops.SLICE: if (cret:=buffers.get(self)) is not None: return cret buf = self.src[0].buffer @@ -1775,10 +1775,14 @@ pm_unbind = PatternMatcher([(UPat(Ops.BIND, name="x"), do_unbind)]) # ctx is source UOp for which we are finding a contiguous view for. used in contiguous_view_offset pm_contiguous_view_offset = PatternMatcher([ - (UPat(Ops.INDEX, src=(UPat(),)), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE))), lambda: UOp.const(0)), - (UPat(Ops.INDEX, src=(UPat(), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda c: c), - (UPat(Ops.INDEX, src=(UPat(), UPat.cvar('c'))), lambda ctx, c: c if resolve(ctx.numel() == 1, False) else None), + # normalize to 1d bitcasts + (UPat(Ops.BITCAST, name="b"), lambda b: b.src[0].flatten().bitcast(b.dtype).reshape(b.shape) if len(b.shape) != 1 else None), + (UPat(Ops.BITCAST, name="b").index(UPat.cvar("c")), lambda ctx, b, c: + b.src[0].flatten().index(UOp.range(ctx.numel() * (osz:=b.element_size())//(isz:=b.src[0].element_size()), 0) + (c * osz//isz)) if b.tag else None), + (UPat(Ops.INDEX, src=(UPat.var("b"),)), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE))), lambda b: b.rtag().index(0)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat(Ops.RANGE)+UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c)), + (UPat(Ops.INDEX, src=(UPat.var("b"), UPat.cvar('c'))), lambda ctx, b, c: b.rtag().index(c) if resolve(ctx.numel() == 1, False) else None), ]) # *** what was symbolic.py *** diff --git a/tinygrad/uop/symbolic.py b/tinygrad/uop/symbolic.py index 1eda875236..f3d78ebfb5 100644 --- a/tinygrad/uop/symbolic.py +++ b/tinygrad/uop/symbolic.py @@ -293,8 +293,8 @@ symbolic = symbolic_simple+commutative+PatternMatcher([ (UPat(Ops.AFTER, name="x"), lambda x: x.replace(src=(x.src[0],)+ tuple(dedup(flatten([(y,) if y.op in {Ops.RANGE, Ops.STORE, Ops.CALL, Ops.FUNCTION, Ops.BARRIER, Ops.END, Ops.LINEAR, Ops.STAGE} else y.src for y in x.src[1:]]))))), - # after with 1 src is just src[0] - (UPat(Ops.AFTER, src=(UPat.var("s"),)), lambda s: s), + # after/end with 1 src is just src[0] + (UPat((Ops.AFTER, Ops.END), src=(UPat.var("s"),)), lambda s: s), ])+div_and_mod_symbolic # ******** we take a small aside to "simplify_valid" to rewrite valids ********