diff --git a/tinygrad/nn/__init__.py b/tinygrad/nn/__init__.py index b813995010..bf6dca75a0 100644 --- a/tinygrad/nn/__init__.py +++ b/tinygrad/nn/__init__.py @@ -10,7 +10,6 @@ class BatchNorm: """ Applies Batch Normalization over a 2D or 3D input. - - Described: https://paperswithcode.com/method/batch-normalization - Paper: https://arxiv.org/abs/1502.03167v3 See: `Tensor.batchnorm` @@ -182,7 +181,6 @@ class GroupNorm: """ Applies Group Normalization over a mini-batch of inputs. - - Described: https://paperswithcode.com/method/group-normalization - Paper: https://arxiv.org/abs/1803.08494v3 ```python exec="true" source="above" session="tensor" result="python" @@ -213,7 +211,6 @@ class InstanceNorm: """ Applies Instance Normalization over a mini-batch of inputs. - - Described: https://paperswithcode.com/method/instance-normalization - Paper: https://arxiv.org/abs/1607.08022v3 ```python exec="true" source="above" session="tensor" result="python" @@ -240,7 +237,6 @@ class LayerNorm: """ Applies Layer Normalization over a mini-batch of inputs. - - Described: https://paperswithcode.com/method/layer-normalization - Paper: https://arxiv.org/abs/1607.06450v1 ```python exec="true" source="above" session="tensor" result="python" @@ -287,7 +283,6 @@ class RMSNorm: """ Applies Root Mean Square Normalization to input. - - Described: https://paperswithcode.com/method/rmsnorm - Paper: https://arxiv.org/abs/1910.07467 ```python exec="true" source="above" session="tensor" result="python" diff --git a/tinygrad/nn/optim.py b/tinygrad/nn/optim.py index 927bd1b291..130e7045a3 100644 --- a/tinygrad/nn/optim.py +++ b/tinygrad/nn/optim.py @@ -76,8 +76,6 @@ def SGD(params: list[Tensor], lr=0.001, momentum=0.0, weight_decay=0.0, nesterov Stochastic Gradient Descent (SGD) optimizer with optional momentum and weight decay. `classic` is a boolean flag that determines whether to use the popular momentum update rule or the classic momentum update rule. - - - Described: https://paperswithcode.com/method/sgd """ return LARS(params, lr, momentum, weight_decay, nesterov, classic, tcoef=0.0, fused=fused) @@ -85,7 +83,6 @@ class LARS(Optimizer): """ Layer-wise Adaptive Rate Scaling (LARS) optimizer with optional momentum and weight decay. - - Described: https://paperswithcode.com/method/lars - Paper: https://arxiv.org/abs/1708.03888v3 """ def __init__(self, params:list[Tensor], lr=0.001, momentum=0.9, weight_decay=1e-4, nesterov=False, classic=True, tcoef=0.001, fused=FUSE_OPTIM): @@ -119,7 +116,6 @@ def AdamW(params: list[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-8, weight_dec """ AdamW optimizer with optional weight decay. - - Described: https://paperswithcode.com/method/adamw - Paper: https://arxiv.org/abs/1711.05101v3 """ return LAMB(params, lr, b1, b2, eps, weight_decay, adam=True, fused=fused) @@ -127,7 +123,6 @@ def Adam(params: list[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-8, fused=FUSE_ """ Adam optimizer. - - Described: https://paperswithcode.com/method/adam - Paper: https://arxiv.org/abs/1412.6980 """ return LAMB(params, lr, b1, b2, eps, 0.0, adam=True, fused=fused) @@ -136,7 +131,6 @@ class LAMB(Optimizer): """ LAMB optimizer with optional weight decay. - - Described: https://paperswithcode.com/method/lamb - Paper: https://arxiv.org/abs/1904.00962 """ def __init__(self, params: list[Tensor], lr=0.001, b1=0.9, b2=0.999, eps=1e-6, weight_decay=0.0, adam=False, fused=FUSE_OPTIM): diff --git a/tinygrad/tensor.py b/tinygrad/tensor.py index f20f1ae527..b62b58d452 100644 --- a/tinygrad/tensor.py +++ b/tinygrad/tensor.py @@ -2332,8 +2332,6 @@ class Tensor(MathTrait): NOTE: unlike PyTorch, this implementation is not limited to only 2d pooling and instead works for any number of dimensions. - See: https://paperswithcode.com/method/average-pooling - ```python exec="true" source="above" session="tensor" result="python" t = Tensor.arange(25).reshape(1, 1, 5, 5) print(t.avg_pool2d().numpy()) @@ -2380,8 +2378,6 @@ class Tensor(MathTrait): NOTE: unlike PyTorch, this implementation is not limited to only 2d pooling and instead works for any number of dimensions. - See: https://paperswithcode.com/method/max-pooling - ```python exec="true" source="above" session="tensor" result="python" t = Tensor.arange(25).reshape(1, 1, 5, 5) print(t.max_pool2d().numpy()) @@ -3010,8 +3006,6 @@ class Tensor(MathTrait): """ Applies the Rectified Linear Unit (ReLU) function element-wise. - - Described: https://paperswithcode.com/method/relu - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).relu().numpy()) ``` @@ -3048,7 +3042,6 @@ class Tensor(MathTrait): Applies the Hardsigmoid function element-wise. NOTE: default `alpha` and `beta` values are taken from torch - - Described: https://paperswithcode.com/method/hard-sigmoid - See: https://pytorch.org/docs/stable/generated/torch.nn.functional.hardsigmoid.html ```python exec="true" source="above" session="tensor" result="python" @@ -3291,7 +3284,6 @@ class Tensor(MathTrait): """ Applies the Exponential Linear Unit (ELU) function element-wise. - - Described: https://paperswithcode.com/method/elu - Paper: https://arxiv.org/abs/1511.07289v5 ```python exec="true" source="above" session="tensor" result="python" @@ -3304,7 +3296,6 @@ class Tensor(MathTrait): """ Applies the Continuously differentiable Exponential Linear Unit (CELU) function element-wise. - - Described: https://paperswithcode.com/method/celu - Paper: https://arxiv.org/abs/1704.07483 ```python exec="true" source="above" session="tensor" result="python" @@ -3317,7 +3308,6 @@ class Tensor(MathTrait): """ Applies the Scaled Exponential Linear Unit (SELU) function element-wise. - - Described: https://paperswithcode.com/method/selu - Paper: https://arxiv.org/abs/1706.02515v5 ```python exec="true" source="above" session="tensor" result="python" @@ -3342,7 +3332,6 @@ class Tensor(MathTrait): """ Applies the Sigmoid Linear Unit (SiLU) function element-wise. - - Described: https://paperswithcode.com/method/silu - Paper: https://arxiv.org/abs/1606.08415 ```python exec="true" source="above" session="tensor" result="python" @@ -3355,7 +3344,6 @@ class Tensor(MathTrait): """ Applies the ReLU6 function element-wise. - - Described: https://paperswithcode.com/method/relu6 - Paper: https://arxiv.org/abs/1704.04861v1 ```python exec="true" source="above" session="tensor" result="python" @@ -3368,7 +3356,6 @@ class Tensor(MathTrait): """ Applies the Hardswish function element-wise. - - Described: https://paperswithcode.com/method/hard-swish - Paper: https://arxiv.org/abs/1905.02244v5 ```python exec="true" source="above" session="tensor" result="python" @@ -3453,8 +3440,6 @@ class Tensor(MathTrait): """ Applies the Hardtanh function element-wise. - - Described: https://paperswithcode.com/method/hardtanh-activation - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-1.5, -1.0, -0.5, 0., 0.5, 1.0, 1.5]).hardtanh().numpy()) ``` @@ -3479,7 +3464,6 @@ class Tensor(MathTrait): """ Applies the Gaussian Error Linear Unit (GELU) function element-wise. - - Described: https://paperswithcode.com/method/gelu - Paper: https://arxiv.org/abs/1606.08415v5 ```python exec="true" source="above" session="tensor" result="python" @@ -3492,8 +3476,6 @@ class Tensor(MathTrait): """ Applies the Sigmoid GELU approximation element-wise. - - Described: https://paperswithcode.com/method/gelu - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).quick_gelu().numpy()) ``` @@ -3504,8 +3486,6 @@ class Tensor(MathTrait): """ Applies the Leaky ReLU function element-wise. - - Described: https://paperswithcode.com/method/leaky-relu - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).leaky_relu().numpy()) ``` @@ -3519,7 +3499,6 @@ class Tensor(MathTrait): """ Applies the Mish function element-wise. - - Described: https://paperswithcode.com/method/mish - Paper: https://arxiv.org/abs/1908.08681v3 ```python exec="true" source="above" session="tensor" result="python" @@ -3532,8 +3511,6 @@ class Tensor(MathTrait): """ Applies the Softplus function element-wise. - - Described: https://paperswithcode.com/method/softplus - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).softplus().numpy()) ``` @@ -3544,8 +3521,6 @@ class Tensor(MathTrait): """ Applies the Softsign function element-wise. - - Described: https://paperswithcode.com/method/softsign - ```python exec="true" source="above" session="tensor" result="python" print(Tensor([-3., -2., -1., 0., 1., 2., 3.]).softsign().numpy()) ``` @@ -3839,7 +3814,6 @@ class Tensor(MathTrait): """ Applies Layer Normalization over a mini-batch of inputs. - - Described: https://paperswithcode.com/method/layer-normalization - Paper: https://arxiv.org/abs/1607.06450v1 ```python exec="true" source="above" session="tensor" result="python" @@ -3858,7 +3832,6 @@ class Tensor(MathTrait): """ Applies Batch Normalization over a mini-batch of inputs. - - Described: https://paperswithcode.com/method/batch-normalization - Paper: https://arxiv.org/abs/1502.03167 ```python exec="true" source="above" session="tensor" result="python" @@ -3883,7 +3856,6 @@ class Tensor(MathTrait): NOTE: dropout is only applied when `Tensor.training` is `True`. - - Described: https://paperswithcode.com/method/dropout - Paper: https://jmlr.org/papers/v15/srivastava14a.html ```python exec="true" source="above" session="tensor" result="python" @@ -3925,7 +3897,6 @@ class Tensor(MathTrait): Computes scaled dot-product attention. `self` is the query tensor, `key` is the key tensor, and `value` is the value tensor. - - Described: https://paperswithcode.com/method/scaled - Paper: https://arxiv.org/abs/1706.03762v7 ```python exec="true" source="above" session="tensor" result="python"