forked from tinygrad/tinygrad
Sieds LyklesandGitHub
1f3950a484
* merge index_dtype_3 * new lowering with Invalid idx * remove that dtype from range * finish merge * annotate better * indentation * dont need that anymore * always process replay for openpilot * more uop_given_valid for idx * valid past index_child * fix bug preventing load getting an alt value * add track_match_stats back in in shapetracker and remove cache * get_valid_idx -> get_valid and get_idx * fix heuristics with new idx * split line * fix typo * fix signature * dont skip idx if stride is 0 the idx may still be invalid * lower const with new valid * delete to_indexed_uops * update shapetracker test * delete axis_is_masked * add cache back * move around comment * fix get_valid bug * move invalid fold to symbolic so its earlier * cleanup * update applying padto to new idx * add unit tests * cleanup * fold line * improve spec * dont try to render Invalid as a float * more consistent invalid index * update some tests * Fold index with true cond * skip test * vconst min max if Invalid in arg * fix signature of UOp.const * add test for min/max of Invalid CONST/VCONST * add InvalidType to as_const signature * is Invalid to isinstance * Add InvalidType to ConstLike * index gate is a where gate * make that a metaclass * fix heurisics for new idx * mypy happy
71 lines
2.4 KiB
Python
71 lines
2.4 KiB
Python
import gc
|
|
from tinygrad import Tensor, UOp, Device
|
|
from tinygrad.shape.shapetracker import views_to_valid_uop
|
|
from tinygrad.engine.realize import method_cache, get_program
|
|
|
|
def uops_allocated(): return sum([isinstance(x, UOp) for x in gc.get_objects()])
|
|
def print_uops():
|
|
for x in gc.get_objects():
|
|
if isinstance(x, UOp): print(x)
|
|
|
|
def start(): pass
|
|
def single_tensor(): Tensor([2])
|
|
def two_plus_two(): Tensor([2])+Tensor([2])
|
|
def two_plus_two_schedule(): (Tensor([2])+Tensor([2])).schedule()
|
|
def two_plus_two_kernel():
|
|
si = (Tensor([2])+Tensor([2])).schedule()[-1]
|
|
get_program(si.ast, Device.default.renderer)
|
|
def two_plus_two_linearize():
|
|
si = (Tensor([2])+Tensor([2])).schedule()[-1]
|
|
get_program(si.ast, Device.default.renderer)
|
|
def two_plus_two_realize(): (Tensor([2])+Tensor([2])).realize()
|
|
def two_plus_two_item(): (Tensor([2])+Tensor([2])).item()
|
|
def gradient_test():
|
|
x = Tensor.eye(3, requires_grad=True)
|
|
y = Tensor([[2.0,0,-2.0]], requires_grad=True)
|
|
z = y.matmul(x).sum()
|
|
z.backward()
|
|
def realized_eye():
|
|
Tensor.eye(3, requires_grad=True).realize()
|
|
def realized_list():
|
|
Tensor([[2.0,0,-2.0]], requires_grad=True).realize()
|
|
def kernel_matmul():
|
|
x = Tensor.eye(3, requires_grad=True)
|
|
y = Tensor([[2.0,0,-2.0]], requires_grad=True)
|
|
z = y.matmul(x)
|
|
si = z.schedule()[-1]
|
|
get_program(si.ast, Device.default.renderer)
|
|
def realized_matmul():
|
|
x = Tensor.eye(3, requires_grad=True)
|
|
y = Tensor([[2.0,0,-2.0]], requires_grad=True)
|
|
z = y.matmul(x)
|
|
Tensor.realize(z)
|
|
def realized_gradient():
|
|
x = Tensor.eye(3, requires_grad=True)
|
|
y = Tensor([[2.0,0,-2.0]], requires_grad=True)
|
|
z = y.matmul(x).sum()
|
|
z.backward()
|
|
Tensor.realize(x, y, z, x.grad, y.grad)
|
|
tests = [start, single_tensor, two_plus_two, two_plus_two_schedule, two_plus_two_kernel,
|
|
two_plus_two_linearize, two_plus_two_realize, two_plus_two_item, gradient_test,
|
|
realized_eye, realized_list, kernel_matmul, realized_matmul, realized_gradient]
|
|
|
|
if __name__ == "__main__":
|
|
gc.disable()
|
|
start_uops = uops_allocated()
|
|
# there's a few consts created as default values
|
|
print_uops()
|
|
for t in tests:
|
|
t()
|
|
|
|
# these caches will keep uops alive
|
|
method_cache.clear()
|
|
views_to_valid_uop.cache_clear()
|
|
|
|
new_uops = uops_allocated()
|
|
gc.collect()
|
|
new_uops_gc = uops_allocated()
|
|
print(f"{t.__name__:30s}: {new_uops:3d} -> {new_uops_gc:3d}")
|
|
assert new_uops == start_uops
|
|
#print_uops()
|