mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-08-29 13:56:06 +00:00
minor shard cleanup [pr] (#8888)
This commit is contained in:
+3
-3
@@ -466,12 +466,12 @@ class UOp(MathTrait, metaclass=UOpMetaClass):
|
||||
if axis is None: lbs = [self] * len(devices)
|
||||
else:
|
||||
if self.shape[axis] % len(devices) != 0: raise RuntimeError(f"multi axis uneven: {self.shape[axis]=} {axis=} {len(devices)=}")
|
||||
# NOTE: this works for both even shards and uneven shards
|
||||
sz = self.shape[axis] // len(devices)
|
||||
sizes = [max(0, min(sz, self.shape[axis] - sz*i)) for i in range(len(devices))]
|
||||
lbs, off = [], 0
|
||||
for sz in sizes:
|
||||
lbs = []
|
||||
for sz,off in zip(sizes, itertools.accumulate(sizes, initial=0)):
|
||||
lbs.append(self.shrink(tuple((0,s) if i != axis else (off,off+sz) for i,s in enumerate(self.shape))))
|
||||
off += sz
|
||||
sharded_lbs = [lb.copy_to_device(d) for lb,d in zip(lbs, devices)]
|
||||
# NOTE: this contiguous is making it impossible for the scheduler to do late const folding
|
||||
return UOp.multi(*[lb.contiguous() for lb in sharded_lbs], axis=axis)
|
||||
|
||||
Reference in New Issue
Block a user