GPT2 inference

import tidygrad
from tidygrad.tensor import Tensor

import tidygrad.func as F
# from tidygrad.func import  embedding, layer_norm, stack, concat
import numpy as np
from lovely_numpy import Lo

from transformers import GPT2Tokenizer
from safetensors import safe_open
# !wget -c https://huggingface.co/gpt2/resolve/main/model.safetensors        -O ./downloaded_weights/gpt2.safetensors
# !wget -c https://huggingface.co/gpt2-medium/resolve/main/model.safetensors -O ./downloaded_weights/gpt2-medium.safetensors
# !wget -c https://huggingface.co/gpt2-large/resolve/main/model.safetensors  -O ./downloaded_weights/gpt2-large.safetensors
# !wget -c https://huggingface.co/gpt2-xl/resolve/main/model.safetensors     -O ./downloaded_weights/gpt2-xl.safetensors
class Gpt2Variant:
    def __init__(self, weight_file, n_head, n_layer):
        self.weight_file = weight_file
        self.n_head = n_head
        self.n_layer = n_layer

gpt2_variants = {
    "gpt2": Gpt2Variant("gpt2.safetensors", 12, 12),
    "gpt2-medium": Gpt2Variant("gpt2-medium.safetensors", 16, 24),
    "gpt2-large": Gpt2Variant("gpt2-large.safetensors", 20, 36),
    "gpt2-xl": Gpt2Variant("gpt2-xl.safetensors", 25, 48),
}

gpt2_variant = "gpt2"
weights_dir = "./downloaded_weights/"

text = "In a hole in the ground there lived a"
tokenizer = GPT2Tokenizer.from_pretrained(gpt2_variant)

# tokens = tokenizer.encode(text)  # returns a list of integers
# tokens = Tensor(tokens)

tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
model = safe_open(weights_dir + gpt2_variants[gpt2_variant].weight_file, framework="np")
Tensor(model.get_tensor("wte.weight"))
Tensor[50257, 768](name="" op=Load):
    v=array[50257, 768] f32 n=38597376 (0.1Gb) x∈[-1.270, 1.785] μ=0.000 σ=0.144
    
import safetensors
import tidygrad.func as F

nn.Module capabilities:

  1. Abstract neural network “modules”, like Linear of Conv2D.

  2. Assignment tracks parameters

class MyModel(Module): def init(): self.w1 = Tensort(…) self.b2 = Tens….

# w1, b1 are tracked as parameters

Then you can call model.parameters() to get a list of parameters.

  1. Save / load weights. Also, count weights.

  2. Fun forward/backward pass on the model.

Pytorch

class nn.Linear(): ….

class Model(nn.Module): init: self.l1 = nn.Linear(…) self.ln = …

forward(x):
    x = self.l1(x) 
    x = self.conv(x)
    ....
    return x

model = Model(…)

y = model(x)

TidyGrad

y = x.mmul(w) + b

class ModelTensors(Dict): init

load(st: safetensor):
    for k in st.keys():
        self.params[k] = st.get_tensor(k)

save():
    .....
    return st

model = ModelTensors

a = model[“h0.ln1.w”] # Returns Tensor a = models.h0.ln1.w

model.parameters() ==> Return list of params

optim = SGD(model.params(), lr=9000)

def transformer()…

loss = transformer(X, y, model) loss.backwards()

optim.step()

def transformer_block(model, i, input, n_head):
    dim = input.shape[-1]
    assert dim % n_head == 0

    ln_1_w = model.get_tensor(f"h.{i}.ln_1.weight")
    ln_1_b = model.get_tensor(f"h.{i}.ln_1.bias")

    ln_1 = F.layer_norm(input, ln_1_w, ln_1_b)
    # ln_1.ad

    attn_w_qkv = model.get_tensor(f"h.{i}.attn.c_attn.weight")
    attn_b_qkv = model.get_tensor(f"h.{i}.attn.c_attn.bias")

    attn_w_q, attn_w_k, attn_w_v = np.split(attn_w_qkv, 3, axis=-1)
    attn_b_q, attn_b_k, attn_b_v = np.split(attn_b_qkv, 3, axis=-1)

    q = ln_1.mmul(attn_w_q) + attn_b_q
    k = ln_1.mmul(attn_w_k) + attn_b_k
    v = ln_1.mmul(attn_w_v) + attn_b_v

    # q_chunked = split_tensor(q, axis=-1, n=12)
    # k_chunked = split_tensor(k, axis=-1, n=12)
    # v_chunked = split_tensor(v, axis=-1, n=12)

    q_chunked = F.stack(q.split(n=n_head, axis=-1), axis=0)
    k_chunked = F.stack(k.split(n=n_head, axis=-1), axis=0)
    v_chunked = F.stack(v.split(n=n_head, axis=-1), axis=0)

    attention = q_chunked.mmul(k_chunked.transpose(-1, -2)) / np.sqrt(dim / n_head)

    mask = np.tril(np.ones(attention.shape), k=0)
    ee = np.exp(attention) * mask

    softmaxed = ee / ee.sum(axis=-1, keepdims=True)

    attention_output = softmaxed.mmul(v_chunked)
    attention_chunks = attention_output.split(axis=0, n=n_head)
    # print("attention_chunks", attention_chunks)

    attention_reshaped = F.concat(attention_chunks, axis=-1)
    attention_reshaped = attention_reshaped[0]
    # print("attention_reshaped", attention_reshaped)

    cproj_w = Tensor(model.get_tensor(f"h.{i}.attn.c_proj.weight"))
    cproj_b = Tensor(model.get_tensor(f"h.{i}.attn.c_proj.bias"))
    # attention_reshaped = Tensor(attention_reshaped_np)

    crosstalk = attention_reshaped.mmul(cproj_w) + cproj_b

    after_residual = crosstalk + input
    # print("after_residual", after_residual)

    ln2_w = Tensor(model.get_tensor(f"h.{i}.ln_2.weight"), name="ln2_w")
    ln2_b = Tensor(model.get_tensor(f"h.{i}.ln_2.bias"), name="ln2_b")

    after_ln2 = F.layer_norm(after_residual, ln2_w, ln2_b)

    mlp_c_fc_w = Tensor(model.get_tensor(f"h.{i}.mlp.c_fc.weight"), name="fc_w")
    mlp_c_fc_b = Tensor(model.get_tensor(f"h.{i}.mlp.c_fc.bias"), name="fc_b")

    after_up = after_ln2.mmul(mlp_c_fc_w) + mlp_c_fc_b
    # print("after_up", after_up)

    after_up_a = F.gelu(after_up)
    # print("after_up_a", after_up_a)

    mlp_c_proj_w = Tensor(model.get_tensor(f"h.{i}.mlp.c_proj.weight"), name="proj_w")
    mlp_c_proj_b = Tensor(model.get_tensor(f"h.{i}.mlp.c_proj.bias"), name="proj_b")

    after_down = after_up_a.mmul(mlp_c_proj_w) + mlp_c_proj_b

    output = after_down + after_residual
    return output

# res = transformer_block(model, 0, embeddings)
def transformer(model, tokens, n_layer, n_head):
    wte = Tensor(model.get_tensor("wte.weight"))
    wpe = Tensor(model.get_tensor("wpe.weight"))

    token_embeddings = F.embedding(wte, tokens)

    positions = np.arange(len(tokens))
    position_embeddings = F.embedding(wpe, positions)

    embeddings = token_embeddings + position_embeddings

    for i in range(n_layer):
        # print("Layer", i)
        embeddings = transformer_block(model, i, embeddings, n_head)
        # print("Embedding out:", embeddings)
        # print(tidygrad.tensor._num_tensors)
        # print(tidygrad.tensor._num_ops)

    ln_f_w = Tensor(model.get_tensor("ln_f.weight"))
    ln_f_b = Tensor(model.get_tensor("ln_f.bias"))

    res = F.layer_norm(embeddings, ln_f_w, ln_f_b)

    return res

# tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

# with tidygrad.no_grad():
#     res = transformer(model, tokens, gpt2_variants[gpt2_variant].n_layer, gpt2_variants[gpt2_variant].n_head)
#     print(res)

# import gc
# del res

# gc.collect()
model.get_tensor("wte.weight").shape
(50257, 768)
text = "In a hole in the ground there lived a"
tokenizer = GPT2Tokenizer.from_pretrained(gpt2_variant)

tokens = tokenizer.encode(text)  # returns a list of integers
print(tokens)
# tokens = list(range(1000))

def gpt2_language_model(model, token_ids, n_layer, n_head):
    wte = Tensor(model.get_tensor("wte.weight").swapaxes(-1, -2))
    res = transformer(model, token_ids, n_layer, n_head)

    res = res[-1, :]
    logits = res.mmul(wte)
    return logits, res

with tidygrad.no_grad():
    logits, res = gpt2_language_model(model, tokens, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)
    print(res)
tokenizer.decode(logits.data.argmax(axis=-1))
[818, 257, 7604, 287, 262, 2323, 612, 5615, 257]
Tensor[768](name="" op=Load):
    v=array[768] f32 3Kb x∈[-50.634, 188.688] μ=0.388 σ=8.312
    
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/activation.py:33: RuntimeWarning: underflow encountered in exp
  self.set_out(1 / (1 + np.exp(-self.args[0].data)))
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/activation.py:33: RuntimeWarning: overflow encountered in exp
  self.set_out(1 / (1 + np.exp(-self.args[0].data)))
' man'
res.data.dtype
dtype('float32')
# import gc
# del logits, res
# gc.collect()
from tqdm.auto import tqdm
Tensor(np.random.randn(5,5)).data.dtype
dtype('float32')
a = np.random.randn(5, 5).astype(np.float32)
b = np.random.randn(5, 5).astype(np.float32)

(a+b).dtype
dtype('float32')
a = np.zeros((1000_000, 1000))
print(tokenizer.special_tokens_map)

text = "<|endoftext|>"
tokens = tokenizer.encode(text)  # returns a list of integers
{'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>'}
[50256]
text = "<|endoftext|> In a hole in the ground there lived a"
# text = "<|endoftext|>"
tokens = tokenizer.encode(text)  # returns a list of integers


print("=== Generating ===")
print("Input: ", tokenizer.decode(tokens))

with tidygrad.no_grad():
    for i in tqdm(range(100)):
        logits, res = gpt2_language_model(model, tokens, n_layer=gpt2_variants[gpt2_variant].n_layer, n_head=gpt2_variants[gpt2_variant].n_head)
        tokens.append(logits.data.argmax(axis=-1))
        del logits, res
        # gc.collect()
        print("Output:", tokenizer.decode(tokens))
=== Generating ===
Input:  <|endoftext|> In a hole in the ground there lived a
Output: <|endoftext|> In a hole in the ground there lived a man
Output: <|endoftext|> In a hole in the ground there lived a man who
Output: <|endoftext|> In a hole in the ground there lived a man who had
Output: <|endoftext|> In a hole in the ground there lived a man who had been
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage.
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Output: <|endoftext|> In a hole in the ground there lived a man who had been killed by a bullet. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of great courage and courage. He was a man of!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/common.py:263: RuntimeWarning: underflow encountered in exp
  self.set_out(np.exp(self.args[0].data))
/tmp/ipykernel_622983/3569323853.py:32: RuntimeWarning: underflow encountered in exp
  ee = np.exp(attention) * mask
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/common.py:263: RuntimeWarning: overflow encountered in exp
  self.set_out(np.exp(self.args[0].data))
/tmp/ipykernel_622983/3569323853.py:32: RuntimeWarning: overflow encountered in exp
  ee = np.exp(attention) * mask
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/common.py:185: RuntimeWarning: invalid value encountered in multiply
  self.set_out(self.args[0].data * self.args[1].data)
/home/xl0/work/projects/grads/tidygrad/tidygrad/ops/activation.py:33: RuntimeWarning: underflow encountered in divide
  self.set_out(1 / (1 + np.exp(-self.args[0].data)))