Tensor.requires_grad -> is_param (#16325)

for optimizer

examples/beautiful_cifar.py

@@ -67,8 +67,8 @@ class ConvGroup:
     self.conv2 = nn.Conv2d(channels_out, channels_out, kernel_size=3, padding=1, bias=False)
     self.norm1 = nn.BatchNorm(channels_out, track_running_stats=False, eps=1e-12, momentum=hyp['net']['batch_norm_momentum'])
     self.norm2 = nn.BatchNorm(channels_out, track_running_stats=False, eps=1e-12, momentum=hyp['net']['batch_norm_momentum'])
-    cast(Tensor, self.norm1.weight).requires_grad = False
-    cast(Tensor, self.norm2.weight).requires_grad = False
+    cast(Tensor, self.norm1.weight).is_param_(False)
+    cast(Tensor, self.norm2.weight).is_param_(False)
   def __call__(self, x:Tensor) -> Tensor:
     x =    self.norm1(self.conv1(x).max_pool2d().float()).cast(dtypes.default_float).quick_gelu()
     return self.norm2(self.conv2(x).float()).cast(dtypes.default_float).quick_gelu() + x

examples/gradaccum_mnist.py

@@ -41,7 +41,7 @@ if __name__ == "__main__":
   Tensor.realize(*params)
 
   # split params (with grads) and buffers (without)
-  params, buffers = partition(params, lambda x: x.requires_grad)
+  params, buffers = partition(params, lambda x: x.is_param)
   print(f"params: {len(params)} buffers: {len(buffers)}")
 
   # optim params

examples/hlb_cifar10.py

@@ -30,9 +30,9 @@ class UnsyncedBatchNorm:
     if affine: self.weight, self.bias = Tensor.ones(sz, dtype=dtypes.float32), Tensor.zeros(sz, dtype=dtypes.float32)
     else: self.weight, self.bias = None, None
 
-    self.running_mean = Tensor.zeros(num_devices, sz, dtype=dtypes.float32, requires_grad=False)
-    self.running_var = Tensor.ones(num_devices, sz, dtype=dtypes.float32, requires_grad=False)
-    self.num_batches_tracked = Tensor.zeros(1, dtype=dtypes.int, requires_grad=False)
+    self.running_mean = Tensor.zeros(num_devices, sz, dtype=dtypes.float32).is_param_(False)
+    self.running_var = Tensor.ones(num_devices, sz, dtype=dtypes.float32).is_param_(False)
+    self.num_batches_tracked = Tensor.zeros(1, dtype=dtypes.int).is_param_(False)
 
   def __call__(self, x:Tensor):
     xr = x.reshape(self.num_devices, -1, *x.shape[1:]).cast(dtypes.float32)
@@ -68,7 +68,7 @@ class UnsyncedBatchNorm:
 class BatchNorm(nn.BatchNorm2d if getenv("SYNCBN") else UnsyncedBatchNorm):
   def __init__(self, num_features):
     super().__init__(num_features, track_running_stats=False, eps=1e-12, momentum=0.85, affine=True)
-    self.weight.requires_grad = False
+    self.weight.is_param_(False)
 
 class ConvGroup:
   def __init__(self, channels_in, channels_out):
@@ -171,7 +171,7 @@ def train_cifar():
     Λ, V = _eigens(_patches(X.float().numpy()))
     W = V/np.sqrt(Λ+1e-2)[:,None,None,None]
 
-    return Tensor(W.astype(np.float32), requires_grad=False).cast(dtypes.default_float)
+    return Tensor(W.astype(np.float32)).cast(dtypes.default_float).is_param_(False)
 
   # ========== Loss ==========
   def cross_entropy(x:Tensor, y:Tensor, reduction:str='mean', label_smoothing:float=0.0) -> Tensor:
@@ -305,7 +305,7 @@ def train_cifar():
   params_bias = []
   params_non_bias = []
   for params in params_dict:
-    if params_dict[params].requires_grad is not False:
+    if params_dict[params].is_param:
       if 'bias' in params:
         params_bias.append(params_dict[params])
       else:

examples/llm.c/train_gpt2.py

@@ -25,7 +25,7 @@ class CausalSelfAttention:
     self.n_embd = config.n_embd
     # not really a 'bias', more of a mask, but following the OpenAI/HF naming though
     self.bias = Tensor.ones(1, 1, config.block_size, config.block_size).tril()
-    self.bias.requires_grad = False
+    self.bias.is_param_(False)
 
   def __call__(self, x:Tensor):
     B, T, C = x.shape

examples/mlperf/initializers.py

@@ -77,11 +77,11 @@ class FrozenBatchNorm2dRetinaNet(nn.BatchNorm2d):
   def __init__(self, sz:int, eps=1e-5, affine=True, track_running_stats=True, momentum=0.1):
     self.eps, self.track_running_stats, self.momentum = eps, track_running_stats, momentum
 
-    self.weight = Tensor.ones(sz, dtype=dtypes.float32, requires_grad=False) if affine else None
-    self.bias = Tensor.zeros(sz, dtype=dtypes.float32, requires_grad=False) if affine else None
+    self.weight = Tensor.ones(sz, dtype=dtypes.float32).is_param_(False) if affine else None
+    self.bias = Tensor.zeros(sz, dtype=dtypes.float32).is_param_(False) if affine else None
 
-    if track_running_stats: self.running_mean, self.running_var = Tensor.zeros(sz, dtype=dtypes.float32, requires_grad=False), Tensor.ones(sz, dtype=dtypes.float32, requires_grad=False)
-    self.num_batches_tracked = Tensor.zeros(1, dtype=dtypes.long, requires_grad=False)
+    if track_running_stats: self.running_mean, self.running_var = Tensor.zeros(sz, dtype=dtypes.float32).is_param_(False), Tensor.ones(sz, dtype=dtypes.float32).is_param_(False)
+    self.num_batches_tracked = Tensor.zeros(1, dtype=dtypes.long).is_param_(False)
 
   def __call__(self, x:Tensor) -> Tensor:
     batch_mean, batch_var = super().calc_stats(x.cast(dtypes.float32))

examples/mlperf/model_train.py

@@ -413,7 +413,7 @@ def train_retinanet():
     layers_to_train = ["layer4", "layer3", "layer2", "layer1", "conv1"][:trainable_layers]
     for k, v in get_state_dict(backbone).items():
       if all([not k.startswith(layer) for layer in layers_to_train]):
-        v.requires_grad = False
+        v.is_param_(False)
 
   def _data_get(it:Iterator[tuple[Tensor, ...]], val:bool=False):
     if val:

examples/mlperf/models/flat_llama.py

@@ -124,9 +124,9 @@ class FlatTransformer:
     self.tok_embeddings = nn.Embedding(vocab_size, dim)
     self.tok_embeddings.weight = Tensor.normal(vocab_size, dim, mean=0.0, std=0.02, dtype=dtypes.bfloat16)
     self.output = Tensor.normal(1, vocab_size, dim, mean=0.0, std=0.02, dtype=dtypes.bfloat16)
-    self.freqs_cis = precompute_freqs_cis(dim // n_heads, max_context * 2, rope_theta).contiguous().requires_grad_(False)
+    self.freqs_cis = precompute_freqs_cis(dim // n_heads, max_context * 2, rope_theta).contiguous().is_param_(False)
 
-    def _amax(): return Tensor.full((), FP8_MAX, dtype=dtypes.float32).contiguous().requires_grad_(False)
+    def _amax(): return Tensor.full((), FP8_MAX, dtype=dtypes.float32).contiguous().is_param_(False)
     names = ["xqkv", "xo", "x2"]
     names += ["x1", "x3"] if SPLIT_W13 else ["x13"]
     self._fp8_amax = {name: [_amax() for _ in range(n_layers)] for name in names}
@@ -135,7 +135,7 @@ class FlatTransformer:
     self._fp8_grad_amax = {name: [_amax() for _ in range(n_layers)] for name in grad_names}
     w_scales = [("wqkv", s_qkv), ("wo", s_o), ("w2", s_2)]
     w_scales += [("w1", s_1), ("w3", s_3)] if SPLIT_W13 else [("w13", s_13)]
-    self._fp8_inv_scale = {name: s.float().contiguous().requires_grad_(False) for name, s in w_scales}
+    self._fp8_inv_scale = {name: s.float().contiguous().is_param_(False) for name, s in w_scales}
 
   def lin_per_layer(self, in_features:int, out_features:int, std:float=0.02):
     if getenv("ZEROS"): w = Tensor.zeros(self.n_layers, out_features, in_features)
@@ -238,9 +238,9 @@ class FlatTransformer:
       for amax_dict in (self._fp8_amax, self._fp8_grad_amax):
         for name in amax_dict:
           for i in range(len(amax_dict[name])):
-            amax_dict[name][i] = amax_dict[name][i].to(device).contiguous().requires_grad_(False)
+            amax_dict[name][i] = amax_dict[name][i].to(device).contiguous().is_param_(False)
       for name in self._fp8_inv_scale:
-        self._fp8_inv_scale[name] = self._fp8_inv_scale[name].to(device).contiguous().requires_grad_(False)
+        self._fp8_inv_scale[name] = self._fp8_inv_scale[name].to(device).contiguous().is_param_(False)
 
   def __call__(self, tokens:Tensor, save:bool=True):
     h = self.tok_embeddings(tokens)
@@ -326,7 +326,7 @@ if __name__ == "__main__":
     if isinstance(x.device, tuple) and x.uop.axis is not None:
       return Tensor.zeros(x.shape, dtype=grad_dtype(x), device=x.device[0]).shard_(x.device, axis=x.uop.axis).contiguous()
     return Tensor.zeros(x.shape, dtype=grad_dtype(x), device=x.device).contiguous()
-  grads = {x:_make_grad(x) for x in state.values() if x.requires_grad}
+  grads = {x:_make_grad(x) for x in state.values() if x.is_param}
 
   fp8_amax = [t for ts in model._fp8_amax.values() for t in ts]
   fp8_grad_amax = [t for ts in model._fp8_grad_amax.values() for t in ts]

extra/models/llama.py

@@ -201,7 +201,7 @@ class Transformer:
     self.tok_embeddings = embedding(vocab_size, dim)
     self.output = nn.Linear(dim, vocab_size, bias=False) if embedding == nn.Embedding else linear(dim, vocab_size, bias=False)
     self.max_context = max_context
-    self.freqs_cis = precompute_freqs_cis(dim // n_heads, self.max_context * 2, rope_theta).contiguous().requires_grad_(False)
+    self.freqs_cis = precompute_freqs_cis(dim // n_heads, self.max_context * 2, rope_theta).contiguous().is_param_(False)
     self.forward_jit = TinyJit(self.forward) if jit else None
 
   def forward(self, tokens:Tensor, start_pos:Union[Variable,int], temperature:float, top_k:int, top_p:float, alpha_f:float, alpha_p:float):

extra/models/transformer.py

@@ -41,7 +41,7 @@ class TransformerBlock:
 class Transformer:
   def __init__(self, syms, maxlen, layers, embed_dim, num_heads, ff_dim):
     self.maxlen, self.syms = maxlen, syms
-    self.embed = Tensor.scaled_uniform(maxlen+syms, embed_dim, requires_grad=False)
+    self.embed = Tensor.scaled_uniform(maxlen+syms, embed_dim).is_param_(False)
     self.tbs = [TransformerBlock(embed_dim, num_heads, ff_dim) for _ in range(layers)]
     self.final = Tensor.scaled_uniform(embed_dim, syms)
 

extra/torch_backend/backend.py

@@ -709,7 +709,7 @@ def wrap_inplace_view_op(k,f):
       views = derived_views(base)
       if views:
         old_base = Tensor(base.uop, device=base.device)
-        old_base.requires_grad = base.requires_grad
+        old_base.is_param = base.is_param
         old_base._views = getattr(base, "_views", set())
         for v in views: v._view_base = old_base
         base._views = set()

test/backend/test_optim.py

@@ -10,17 +10,20 @@ x_init = np.random.randn(1,4).astype(np.float32)
 W_init = np.random.randn(4,4).astype(np.float32)
 m_init = np.random.randn(1,4).astype(np.float32)
 
+def _param(tensor, val):
+  return tensor(val, requires_grad=True) if tensor is torch.tensor else tensor(val)
+
 class TeenyNet:
   def __init__(self, tensor):
-    self.x = tensor(x_init.copy(), requires_grad=True)
-    self.W = tensor(W_init.copy(), requires_grad=True)
+    self.x = _param(tensor, x_init.copy())
+    self.W = _param(tensor, W_init.copy())
   def forward(self):
     return (self.x * self.W).sum()
 
 class TinyNet:
   def __init__(self, tensor):
-    self.x = tensor(x_init.copy(), requires_grad=True)
-    self.W = tensor(W_init.copy(), requires_grad=True)
+    self.x = _param(tensor, x_init.copy())
+    self.W = _param(tensor, W_init.copy())
     self.m = tensor(m_init.copy())
 
   def forward(self):

test/backend/test_setitem.py

@@ -301,26 +301,26 @@ class TestSetitem(unittest.TestCase):
     self.assertListEqual(z[6:7].tolist(), [3])
 
 class TestWithGrad(unittest.TestCase):
-  def test_no_requires_grad_works(self):
+  def test_basic_setitem_works(self):
     z = Tensor.rand(8, 8)
     x = Tensor.rand(8)
     z[:3] = x
 
-  def test_set_with_requires_grad(self):
+  def test_set_backward(self):
     z = Tensor.ones(8, 8)
     x = Tensor.rand(8, 8)
     z[:] = x
     z.sum().backward()
     np.testing.assert_allclose(x.grad.numpy(), np.ones((8, 8)))
 
-  def test_set_nonleaf_requires_grad(self):
+  def test_set_nonleaf_backward(self):
     x = Tensor([1.0, 2.0, 3.0, 4.0])
     z = x * 2
     z[:2] = Tensor([10.0, 20.0])
     z.sum().backward()
     np.testing.assert_allclose(x.grad.numpy(), [0, 0, 2, 2])
 
-  def test_set_overlapping_requires_grad(self):
+  def test_set_overlapping_backward(self):
     z = Tensor.zeros(6)
     x = Tensor.ones(4)
     y = Tensor.ones(4) * 2
@@ -330,7 +330,7 @@ class TestWithGrad(unittest.TestCase):
     np.testing.assert_allclose(x.grad.numpy(), [1, 1, 0, 0])
     np.testing.assert_allclose(y.grad.numpy(), np.ones(4))
 
-  def test_set_iadd_requires_grad(self):
+  def test_set_iadd_backward(self):
     z = Tensor([1.0, 2.0, 3.0, 4.0])
     x = Tensor([10.0, 20.0])
     z[:2] += x

test/backend/test_tensor.py

@@ -192,9 +192,9 @@ class TestTinygrad(unittest.TestCase):
     def test_tinygrad():
       w1 = Tensor(init).clone()
       w2 = Tensor(init).clone()
-      assert w1.requires_grad is True and w2.requires_grad is True
+      assert w1.is_param is True and w2.is_param is True
       nn.optim.SGD([w1, w2], lr=0.01)
-      assert w1.requires_grad is True and w2.requires_grad is True
+      assert w1.is_param is True and w2.is_param is True
       out = w1.add(w2)
       out.backward()
       return w1.grad.numpy(), w2.grad.numpy()
@@ -260,10 +260,6 @@ class TestTinygrad(unittest.TestCase):
   def test_rand_rejects_unknown_kwargs(self):
     with self.assertRaises(TypeError): Tensor.rand(5, generator="foo")
 
-  def test_randperm_requires_grad(self):
-    self.assertIs(Tensor.randperm(5, requires_grad=True).requires_grad, True)
-    self.assertIs(Tensor.randperm(5, requires_grad=False).requires_grad, False)
-
   def test_randn_isnt_inf_on_zero(self):
     # simulate failure case of rand handing a zero to randn
     original_rand, Tensor.rand = Tensor.rand, Tensor.zeros
@@ -568,26 +564,26 @@ class TestTinygrad(unittest.TestCase):
 class TestMoveTensor(unittest.TestCase):
   d0, d1 = f"{Device.DEFAULT}:0", f"{Device.DEFAULT}:1"
   @given(strat.sampled_from([d0, d1]), strat.sampled_from([d0, d1]),
-         strat.sampled_from([dtypes.float16, dtypes.float32]), strat.sampled_from([True, False, None]))
-  def test_to_preserves(self, src, dest, dtype, requires_grad):
+         strat.sampled_from([dtypes.float16, dtypes.float32]), strat.sampled_from([True, False]))
+  def test_to_preserves(self, src, dest, dtype, is_param):
     if dtype not in Device[Device.DEFAULT].renderer.supported_dtypes():
       return
-    s = Tensor([1, 2, 3], device=src, dtype=dtype, requires_grad=requires_grad)
-    if requires_grad: s.sum().backward()
+    s = Tensor([1, 2, 3], device=src, dtype=dtype).is_param_(is_param)
+    if is_param: s.sum().backward()
     t = s.to(dest)
     np.testing.assert_equal(s.numpy(), t.numpy())
     assert s.dtype == t.dtype
-    assert s.requires_grad == t.requires_grad
-    if requires_grad:
+    assert s.is_param == t.is_param
+    if is_param:
       np.testing.assert_equal(s.grad.numpy(), t.grad.numpy())
 
-  @given(strat.sampled_from([dtypes.float16, dtypes.float32]), strat.sampled_from([True, False, None]))
-  def test_shard_preserves(self, dtype, requires_grad):
-    s = Tensor([1, 2, 3], dtype=dtype, requires_grad=requires_grad)
+  @given(strat.sampled_from([dtypes.float16, dtypes.float32]), strat.sampled_from([True, False]))
+  def test_shard_preserves(self, dtype, is_param):
+    s = Tensor([1, 2, 3], dtype=dtype).is_param_(is_param)
     t = s.shard((f"{Device.DEFAULT}:0", f"{Device.DEFAULT}:1"))
     np.testing.assert_equal(s.numpy(), t.numpy())
     assert s.dtype == t.dtype
-    assert s.requires_grad == t.requires_grad
+    assert s.is_param == t.is_param
 
   @given(strat.sampled_from([d0, d1]))
   def test_same_dev(self, dev):

test/null/test_tensor.py

@@ -36,7 +36,6 @@ class TestInferenceMode(unittest.TestCase):
     assert tmp.grad is None
     assert mm.grad is None
     assert W.grad is None
-    assert W.requires_grad
 
   def test_no_grad_mode_context_manager(self):
     x = Tensor(x_init)

tinygrad/nn/__init__.py

@@ -35,8 +35,8 @@ class BatchNorm:
     self.weight: Tensor|None = Tensor.ones(sz) if affine else None
     self.bias: Tensor|None = Tensor.zeros(sz) if affine else None
 
-    self.num_batches_tracked = Tensor.zeros(dtype='long', requires_grad=False)
-    if track_running_stats: self.running_mean, self.running_var = Tensor.zeros(sz, requires_grad=False), Tensor.ones(sz, requires_grad=False)
+    self.num_batches_tracked = Tensor.zeros(dtype='long').is_param_(False)
+    if track_running_stats: self.running_mean, self.running_var = Tensor.zeros(sz).is_param_(False), Tensor.ones(sz).is_param_(False)
 
   def calc_stats(self, x:Tensor) -> tuple[Tensor, Tensor]:
     shape_mask: list[int] = [1, -1, *([1]*(x.ndim-2))]

tinygrad/nn/optim.py

@@ -10,9 +10,9 @@ class Optimizer:
   """
   def __init__(self, params: list[Tensor], lr: float, device=None, fused=FUSE_OPTIM):
     if lr < 0: raise ValueError(f"Invalid learning rate: {lr}")
-    self.params: list[Tensor] = dedup([x for x in params if x.requires_grad])
+    self.params: list[Tensor] = dedup([x for x in params if x.is_param])
     assert len(self.params) != 0, "optimizer must have at least one param"
-    self.buffers: list[Tensor] = dedup([x for x in params if not x.requires_grad])   # buffers are still realized
+    self.buffers: list[Tensor] = dedup([x for x in params if not x.is_param])   # buffers are still realized
     self.device = device or self.params[0].device
     self.param_dtype = to_dtype(getenv("OPTIM_DTYPE", "float32"))
     self.fused = fused
@@ -154,7 +154,7 @@ class LAMB(Optimizer):
     if weight_decay < 0: raise ValueError(f"Invalid weight_decay value: {weight_decay}")
     super().__init__(params, lr, device, fused)
     self.b1, self.b2, self.eps, self.wd, self.adam = b1, b2, eps, weight_decay, adam
-    self.b1_t, self.b2_t = (Tensor.ones((1,), dtype=dtypes.float32, device=self.device, requires_grad=False) for _ in [b1, b2])
+    self.b1_t, self.b2_t = (Tensor.ones((1,), dtype=dtypes.float32, device=self.device).is_param_(False) for _ in [b1, b2])
     self.m = self._new_optim_param()
     self.v = self._new_optim_param()
 

tinygrad/tensor.py

@@ -88,11 +88,11 @@ class Tensor(OpMixin):
   np.set_printoptions(precision=4)
   ```
   """
-  __slots__ = "uop", "requires_grad", "grad"
+  __slots__ = "uop", "is_param", "grad"
   training: ClassVar[bool] = False
 
   def __init__(self, data:ConstType|bytes|list|tuple|UOp|'numpy.ndarray'|pathlib.Path|None,
-               device:str|tuple|list|None=None, dtype:DTypeLike|None=None, requires_grad:bool=True):
+               device:str|tuple|list|None=None, dtype:DTypeLike|None=None):
     if device is None:
       if isinstance(data, pathlib.Path): device = f"DISK:{data.resolve()}"  # keep it on the disk if device is None
       elif isinstance(data, UOp): device = data.device
@@ -103,7 +103,7 @@ class Tensor(OpMixin):
     # tensors can have gradients if you have called .backward
     self.grad:Tensor|None = None
 
-    self.requires_grad:bool = requires_grad
+    self.is_param:bool = True
 
     # create a UOp from the different types of inputs
     if isinstance(data, UOp):
@@ -150,15 +150,14 @@ class Tensor(OpMixin):
     if TRACEMETA >= 1 and (metadata:=_METADATA.get()) is not None: all_metadata[new_uop] = (metadata,)
     # directly create the Tensor
     ret = Tensor.__new__(Tensor)
-    ret.uop, ret.grad = new_uop, None
-    ret.requires_grad = any(t.requires_grad for t in srcs)
+    ret.uop, ret.grad, ret.is_param = new_uop, None, True
     # add to all_tensors after construction succeeds
     all_tensors[weakref.ref(ret)] = None
     return ret
 
   # alu and const_like are used by the mixins
   def alu(self, op: Ops, *src: Tensor) -> Tensor: return self._apply_uop(lambda *u: u[0].alu(op, *u[1:]), *src)
-  def const_like(self, b:ConstType) -> Tensor: return Tensor(self.uop.const_like(b), requires_grad=False)
+  def const_like(self, b:ConstType) -> Tensor: return Tensor(self.uop.const_like(b))
   @staticmethod
   def const(dtype:DType, b:ConstType|UOp, device:str|tuple[str, ...]|None=None) -> Tensor:
     return Tensor(b if isinstance(b, UOp) else UOp.const(dtype, b, device))
@@ -168,9 +167,8 @@ class Tensor(OpMixin):
     dtype, device = kwargs.pop("dtype", None), kwargs.pop("device", None)
     return Tensor(UOp.unique_const(fill_value, dtype, device), **kwargs)
 
-  def requires_grad_(self, requires_grad:bool=True) -> Tensor:
-    if requires_grad and self.uop.op is Ops.CONST: self.replace(self.clone())
-    self.requires_grad = requires_grad
+  def is_param_(self, is_param:bool=True) -> Tensor:
+    self.is_param = is_param
     return self
 
   class train(ContextDecorator):
@@ -362,7 +360,7 @@ class Tensor(OpMixin):
     """
     ret = Tensor(self.uop.clone(device=device))
     if self.grad is not None: ret.grad = self.grad.clone(device=device)
-    return ret
+    return ret.is_param_(self.is_param)
 
   def to(self, device:str|tuple[str, ...]|None) -> Tensor:
     """
@@ -372,7 +370,7 @@ class Tensor(OpMixin):
     if (device:=canonicalize_device(device)) == self.device: return self
     ret = Tensor(self.uop.copy_to_device(device))
     if self.grad is not None: ret.grad = self.grad.to(device)
-    return ret
+    return ret.is_param_(self.is_param)
 
   def to_(self, device:str|tuple[str, ...]|None) -> Tensor:
     """
@@ -396,7 +394,7 @@ class Tensor(OpMixin):
     if len(devices) == 1: return self.to(devices[0])
     devices = cast(tuple[str, ...], canonicalize_device(devices))
     uop = self.uop.shard(devices, self._resolve_dim(axis)) if axis is not None else self.uop.copy_to_device(devices)
-    return Tensor(uop)
+    return Tensor(uop).is_param_(self.is_param)
 
   def shard_(self, devices:tuple[str, ...], axis:int|None=None) -> Tensor:
     """
@@ -566,7 +564,7 @@ class Tensor(OpMixin):
     return Tensor._device_seeds[device], low.cat(high)
 
   @staticmethod
-  def rand(*shape, device:str|None=None, dtype:DTypeLike|None=None, requires_grad:bool=True, contiguous:bool=True) -> Tensor:
+  def rand(*shape, device:str|None=None, dtype:DTypeLike|None=None, contiguous:bool=True) -> Tensor:
     """
     Creates a tensor with the given shape, filled with random values from a uniform distribution over the interval `[0, 1)`.
 
@@ -589,7 +587,7 @@ class Tensor(OpMixin):
     num = ceildiv(numel * dt.itemsize, 4)
     key, counter = Tensor._next_counter(device, num)
     bits = Tensor.random_bits(key, counter, num)
-    out = Tensor._bits_to_rand(bits, shape, dt).requires_grad_(requires_grad)
+    out = Tensor._bits_to_rand(bits, shape, dt)
     return out.contiguous() if contiguous else out
 
   # ***** creation helper functions *****
@@ -600,9 +598,9 @@ class Tensor(OpMixin):
     assert isinstance(self.device, tuple), f"_multi_like needs a multi device tensor, got {self.device}"
     if self.uop.axis is None: return fxn(self.shape, *args, dtype=dtype, **kwargs).shard(self.device)
     stacked = UOp.mstack(*[fxn(self.uop.shard_shape, *args, device=d, dtype=dtype, **kwargs).uop for d in self.device])
-    return Tensor(stacked.multi(self.uop.axis), requires_grad=kwargs.get("requires_grad", True))
+    return Tensor(stacked.multi(self.uop.axis))
 
-  def full_like(self, fill_value:ConstType, dtype=None, device=None, requires_grad:bool=False) -> Tensor:
+  def full_like(self, fill_value:ConstType, dtype=None, device=None) -> Tensor:
     """
     Creates a tensor with the same shape as `self`, filled with the given value.
     If `dtype` is not specified, the dtype of `self` is used.
@@ -614,9 +612,9 @@ class Tensor(OpMixin):
     print(Tensor.full_like(t, 42).numpy())
     ```
     """
-    if device is None: return super().full_like(fill_value, dtype).requires_grad_(requires_grad)
+    if device is None: return super().full_like(fill_value, dtype)
     if isinstance(self.device, tuple): raise RuntimeError("cannot specify `device` on `full_like` of a multi device tensor")
-    return Tensor.full(self.shape, fill_value, dtype=dtype or self.dtype, device=device).requires_grad_(requires_grad)
+    return Tensor.full(self.shape, fill_value, dtype=dtype or self.dtype, device=device)
 
   def rand_like(self, **kwargs) -> Tensor:
     """
@@ -635,7 +633,7 @@ class Tensor(OpMixin):
 
   # ***** random functions *****
 
-  def randn_like(self, dtype:DTypeLike|None=None, requires_grad:bool=True, **kwargs) -> Tensor:
+  def randn_like(self, dtype:DTypeLike|None=None, **kwargs) -> Tensor:
     """
     Creates a tensor with the same shape and sharding as `self`, filled with random values from a normal distribution with mean 0 and variance 1.
 
@@ -649,10 +647,10 @@ class Tensor(OpMixin):
     """
     src = self.stack(self).rand_like(**{**kwargs, "dtype": dtypes.float32})
     # https://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform
-    return (src[0].mul(2*math.pi).cos().mul((1 - src[1]).log().mul(-2).sqrt()).cast(dtype or self.dtype)).requires_grad_(requires_grad)
+    return src[0].mul(2*math.pi).cos().mul((1 - src[1]).log().mul(-2).sqrt()).cast(dtype or self.dtype)
 
   @staticmethod
-  def randn(*shape, dtype:DTypeLike|None=None, requires_grad:bool=True, **kwargs) -> Tensor:
+  def randn(*shape, dtype:DTypeLike|None=None, **kwargs) -> Tensor:
     """
     Creates a tensor with the given shape, filled with random values from a normal distribution with mean `0` and standard deviation `1`.
     If `dtype` is not specified, the default type is used.
@@ -687,7 +685,7 @@ class Tensor(OpMixin):
     return Tensor.uniform(*shape, low=low, high=high, dtype=dtype, **kwargs)
 
   @staticmethod
-  def normal(*shape, mean=0.0, std=1.0, requires_grad:bool=True, **kwargs) -> Tensor:
+  def normal(*shape, mean=0.0, std=1.0, **kwargs) -> Tensor:
     """
     Creates a tensor with the given shape, filled with random values from a normal distribution with the given `mean` and standard deviation `std`.
     Requires `std >= 0`.
@@ -701,10 +699,10 @@ class Tensor(OpMixin):
     ```
     """
     if std < 0: raise ValueError(f"Tensor.normal requires std >= 0, got {std=}")
-    return (std * Tensor.randn(*shape, **kwargs) + mean).requires_grad_(requires_grad)
+    return std * Tensor.randn(*shape, **kwargs) + mean
 
   @staticmethod
-  def uniform(*shape, low=0.0, high=1.0, dtype:DTypeLike|None=None, requires_grad:bool=True, **kwargs) -> Tensor:
+  def uniform(*shape, low=0.0, high=1.0, dtype:DTypeLike|None=None, **kwargs) -> Tensor:
     """
     Creates a tensor with the given shape, filled with random values from a uniform distribution over the interval `[low, high)`.
     Requires `low < high`.
@@ -719,7 +717,7 @@ class Tensor(OpMixin):
     """
     if not all_int(shape:=argfix(*shape)) or not all(s >= 0 for s in shape): raise ValueError(f"invalid input {shape=}")
     if low >= high: raise ValueError(f"Tensor.uniform requires low < high, got {low=}, {high=}")
-    return (((high-low) * Tensor.rand(*shape, **kwargs)).cast(dtype or dtypes.default_float) + low).requires_grad_(requires_grad)
+    return ((high-low) * Tensor.rand(*shape, **kwargs)).cast(dtype or dtypes.default_float) + low
 
   @staticmethod
   def scaled_uniform(*shape, **kwargs) -> Tensor:
@@ -795,7 +793,7 @@ class Tensor(OpMixin):
     print(Tensor.randperm(6).numpy())
     ```
     """
-    return Tensor.rand(n, device=device, **kwargs).argsort().cast(dtype).requires_grad_(kwargs.get("requires_grad", True))
+    return Tensor.rand(n, device=device, **kwargs).argsort().cast(dtype)
 
   def multinomial(self:Tensor, num_samples:int = 1, replacement:bool = False) -> Tensor:
     """
@@ -1016,7 +1014,7 @@ class Tensor(OpMixin):
       if any(self.uop in t.uop.backward_slice_with_self and t.uop.base is not shared for tref in all_tensors
              if (t:=tref()) is not None and t is not self and t.uop is not v_uop and t.uop not in v_bw):
         raise RuntimeError("can't setitem on a tensor with other uses")
-    if not self.uop.base.is_realized and self.is_floating_point() and (self.requires_grad or (isinstance(v, Tensor) and v.requires_grad)):
+    if not self.uop.base.is_realized and self.is_floating_point():
       if not isinstance(v, Tensor): v = Tensor(v, device=self.device, dtype=self.dtype)
       # __iadd__/__isub__ creates AFTER(view, STORE(view, computed)); unwrap to get the computed value
       if v.uop.op is Ops.AFTER and any(s.op is Ops.STORE for s in v.uop.src[1:]): v = v._apply_uop(lambda x: x.src[1].src[1])