From c26ed64cef0d172fddd773ed6095daafe137809b Mon Sep 17 00:00:00 2001 From: George Hotz Date: Mon, 6 Jun 2022 09:31:48 -0700 Subject: [PATCH] slightly cleaner graph --- tinygrad/optim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tinygrad/optim.py b/tinygrad/optim.py index b86840e274..5f4298e758 100644 --- a/tinygrad/optim.py +++ b/tinygrad/optim.py @@ -27,7 +27,7 @@ class RMSprop(Optimizer): def step(self): for i, t in enumerate(self.params): - self.v[i] = self.decay * self.v[i] + (1.0 - self.decay) * t.grad * t.grad + self.v[i] = self.decay * self.v[i] + (1.0 - self.decay) * (t.grad * t.grad) t -= (t.grad * self.lr).mul(self.v[i].add(self.eps).pow(-0.5)) class Adam(Optimizer): @@ -43,5 +43,5 @@ class Adam(Optimizer): a = self.lr * ((1.0 - self.b2**self.t)**0.5) / (1.0 - self.b1**self.t) for i, t in enumerate(self.params): self.m[i] = self.b1 * self.m[i] + (1.0 - self.b1) * t.grad - self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * t.grad * t.grad + self.v[i] = self.b2 * self.v[i] + (1.0 - self.b2) * (t.grad * t.grad) t -= a * self.m[i].mul(self.v[i].add(self.eps).pow(-0.5))