notes

extra/upcasted_warps.py

@@ -6,12 +6,13 @@ from tinygrad.opt.kernel import OptOps, Opt
 from tinygrad.engine.realize import get_program
 
 if __name__ == "__main__":
-
   renderer = Device.default.renderer
   N = 64
+
+  """
   a = Tensor.empty(N,N)
 
-  out = a + 1
+  out = (a + 1) #.sum(axis=2)
   ast = out.schedule()[-1].ast
   opts = tuple()
   opts += (Opt(OptOps.UPCAST, 0, 32),)
@@ -19,15 +20,20 @@ if __name__ == "__main__":
   ast = get_optimized_ast(ast, renderer)
   prg = get_program(ast, renderer)
   print(prg.src)
+  """
 
   # how you split the store determines everything if you don't allow cross warp comms.
   # actually not everything, there's also the split before the horizontal (unrolled) reduces
 
   # new flow
   #  - pull out any dimensions from the store that you want to upcast.
-  #  - decide how you want to assign them to registers. GPUs have a 128-byte memory LOAD/STORE which loads into 4 regs. see BUFFER_LOAD_B128
+  #  - decide how you want to assign them to registers. GPUs have a 512-byte memory LOAD/STORE which loads into 4 regs. see BUFFER_LOAD_B128
   #    - the loads and stores can be shuffled, but only in restrictive ways. in kernels without reduces, the store determines everything
-  #  - in kernels with reduces, you now have more flexibility
+  #    - it loads 16 bytes from up 32 different places = 512 bytes
+  #  - in kernels with reduces, you now have more flexibility. the final target of the reduce must be what is stored
+  #    - warp dimensions can be in the reduce (this is GROUP)
+
+  # every dimension can be assigned to <global, local, loop, upcast, warp>
 
   """
   out = a.sum(axis=1)
@@ -52,7 +58,6 @@ if __name__ == "__main__":
   """
 
   # gemm
-  """
   b = Tensor.empty(N,N)
   # metal TC
   #opts = (Opt(OptOps.UPCAST, 0, 2), # not the warp
@@ -66,4 +71,3 @@ if __name__ == "__main__":
   ast = get_optimized_ast(ast, renderer)
   prg = get_program(ast, renderer)
   print(prg.src)
-  """