minor shard cleanup [pr] (#8888)

This commit is contained in:
chenyu
2025-02-04 13:22:31 -05:00
committed by GitHub
parent 6ec7f1b00f
commit 61de654efa
+3 -3
View File
@@ -466,12 +466,12 @@ class UOp(MathTrait, metaclass=UOpMetaClass):
if axis is None: lbs = [self] * len(devices)
else:
if self.shape[axis] % len(devices) != 0: raise RuntimeError(f"multi axis uneven: {self.shape[axis]=} {axis=} {len(devices)=}")
# NOTE: this works for both even shards and uneven shards
sz = self.shape[axis] // len(devices)
sizes = [max(0, min(sz, self.shape[axis] - sz*i)) for i in range(len(devices))]
lbs, off = [], 0
for sz in sizes:
lbs = []
for sz,off in zip(sizes, itertools.accumulate(sizes, initial=0)):
lbs.append(self.shrink(tuple((0,s) if i != axis else (off,off+sz) for i,s in enumerate(self.shape))))
off += sz
sharded_lbs = [lb.copy_to_device(d) for lb,d in zip(lbs, devices)]
# NOTE: this contiguous is making it impossible for the scheduler to do late const folding
return UOp.multi(*[lb.contiguous() for lb in sharded_lbs], axis=axis)