Tidy Utils

grad_check

 grad_check (func, inputs, params:tuple=(), eps=1e-05, n=1000,
             verbose=False)

from lovely_numpy import Lo
from tidygrad import Tensor

Lo((np.random.randn(32, 28 * 28) @ (np.random.randn(28 * 28, 100) * 0.1) + np.random.randn(100)) @ (np.random.randn(100, 10) * 0.1))

array[32, 10] n=320 (2.5Kb) x∈[-7.871, 6.829] μ=-0.001 σ=2.903

i = 0

i += 1
np.random.seed(i)

x = Tensor(np.random.randn(32, 28 * 28), "X")
# Create a 1-hot encoded tensor with 1 random 1
y = np.zeros((32, 10))
y[np.arange(32), np.random.choice(10, 32)] = 1
y = Tensor(y, "y")

w1 = Tensor(np.random.randn(28 * 28, 100) * 0.1, "w1", requires_grad=True)
b1 = Tensor(np.random.randn(100), "b1", requires_grad=True)
w2 = Tensor(np.random.randn(100, 10) * 0.1, "w2", requires_grad=True)

def NN(inputs, params: tuple):
    x, y = inputs
    w1, b1, w2 = params
    z1 = x.mmul(w1, "tmp").add(b1, "z1")
    a1 = tg.sigmoid(z1)
    z2 = a1.mmul(w2)

    loss = -tg.BCE_loss(z2, y).sum("loss")

    return loss

debug = []
loss = NN(inputs=(x, y), params=(w1, b1, w2))

loss.backward()

# grad_check(NN, (x, y), (w1, b1, w2))

Max fractional gradient difference for w2: 0.0011%
Max fractional gradient difference for b1: 0.0010%
Max fractional gradient difference for w1: 0.0159%