lecture_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. imports \n",
    "\n",
    "import math \n",
    "import random \n",
    "from dataclasses import dataclass\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn \n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from biotite.sequence.io.fasta import FastaFile\n",
    "\n",
    "\n",
    "torch.manual_seed(12)\n",
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. define a Torch dataset class for our data \n",
    "\n",
    "class ProteinDataset(Dataset):\n",
    "    \"\"\"Dataset for protein sequences with character-level tokenization\"\"\"\n",
    "\n",
    "    def __init__(self, proteins, chars, max_protein_length):\n",
    "        \"\"\"Create a dataset \n",
    "        \n",
    "        proteins: list of str, protein sequences\n",
    "        chars: list of str, all the characters in the vocabulary\n",
    "        max_protein_length: int, the length of the longest sequence \n",
    "        \"\"\"\n",
    "        self.proteins = proteins\n",
    "        self.chars = chars\n",
    "        self.max_protein_length = max_protein_length\n",
    "        self.stoi = {ch: i + 1 for i, ch in enumerate(chars)}\n",
    "        self.itos = {i: s for s, i in self.stoi.items()} # inverse mapping\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.proteins)\n",
    "\n",
    "    def contains(self, protein):\n",
    "        return protein in self.proteins\n",
    "\n",
    "    def get_vocab_size(self):\n",
    "        return len(self.chars) + 1 # all the possible characters and special 0 token\n",
    "\n",
    "    def get_output_length(self):\n",
    "        return self.max_protein_length + 1 # <START> token followed by proteins\n",
    "\n",
    "    def encode(self, protein):\n",
    "        ix = torch.tensor([self.stoi[w] for w in protein], dtype=torch.long)\n",
    "        return ix\n",
    "\n",
    "    def decode(self, ix):\n",
    "        word = ''.join(self.itos[i] for i in ix)\n",
    "        return word\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        protein = self.proteins[idx]\n",
    "        ix = self.encode(protein)\n",
    "        x = torch.zeros(self.max_protein_length + 1, dtype=torch.long)\n",
    "        y = torch.zeros(self.max_protein_length + 1, dtype=torch.long)\n",
    "        x[1:1+len(ix)] = ix\n",
    "        y[:len(ix)] = ix\n",
    "        y[len(ix)+1:] = -1 # index -1 will mask the loss at the inactive locations\n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0, 11,  2, 10, 10, 16, 10,  1,  1,  1, 17, 18,  1,  1, 15, 15, 17, 13,\n",
       "        10, 15, 10, 10,  6, 15,  6, 10,  1,  1,  1, 11, 16, 17,  1,  6, 13, 10,\n",
       "         9, 16, 18])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. dataset example, some short proteins of length 38 \n",
    "\n",
    "proteins = [\n",
    "    \"MCLLSLAAATVAARRTPLRLLGRGLAAAMSTAGPLKSV\", \n",
    "    \"MSSQIKKSKTTTKKLVKSAPKSVPNAAADDQIFCCQFE\", \n",
    "    \"MCLLSLAAATVAARRTPLRLLGRGLAAAMSTAGPLKSV\", \n",
    "]\n",
    "\n",
    "chars = \"ACDEFGHIKLMNPQRSTVWY\"\n",
    "\n",
    "max_length = 38 \n",
    "\n",
    "dataset = ProteinDataset(proteins, chars, max_length)\n",
    "\n",
    "x, y = dataset[0]\n",
    "\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. now wrap this logic into a nice function \n",
    "\n",
    "def create_datasets(input_file):\n",
    "    \"\"\"Create train and test datasets from a FASTA file (90/10 split)\"\"\"\n",
    "\n",
    "    # preprocessing of the input text file\n",
    "    proteins = []\n",
    "    fasta_file = FastaFile.read(input_file) \n",
    "    for header, sequence in fasta_file.items():\n",
    "        proteins.append(sequence)\n",
    "    max_protein_length = max(len(w) for w in proteins)\n",
    "\n",
    "    # partition the input data into a training and the test set\n",
    "    test_set_size = int(len(proteins) * 0.1) # 10% of the training set\n",
    "    rp = torch.randperm(len(proteins)).tolist()\n",
    "    train_proteins = [proteins[i] for i in rp[:-test_set_size]]\n",
    "    test_proteins = [proteins[i] for i in rp[-test_set_size:]]\n",
    "    print(f\"Split up the dataset into {len(train_proteins)} training examples and {len(test_proteins)} test examples\")\n",
    "\n",
    "    chars = sorted(list(set(''.join(proteins)))) # all the possible characters\n",
    "    tokens = sum(len(w) for w in proteins)\n",
    "    \n",
    "    print(f\"Number of examples in the dataset: {len(proteins)}\")\n",
    "    print(f\"Max protein length: {max_protein_length}\")\n",
    "    print(f\"Number of unique characters in the vocabulary: {len(chars)}\")\n",
    "    print(f\"Vocabulary (amino acids): {''.join(chars)}\")\n",
    "    print(f\"Total tokens: {tokens}\")\n",
    "\n",
    "    # wrap in dataset objects\n",
    "    train_dataset = ProteinDataset(train_proteins, chars, max_protein_length)\n",
    "    test_dataset = ProteinDataset(test_proteins, chars, max_protein_length)\n",
    "\n",
    "    return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split up the dataset into 24102 training examples and 2677 test examples\n",
      "Number of examples in the dataset: 26779\n",
      "Max protein length: 127\n",
      "Number of unique characters in the vocabulary: 20\n",
      "Vocabulary (amino acids): ACDEFGHIKLMNPQRSTVWY\n",
      "Total tokens: 2519424\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([ 0, 11, 10, 15,  9,  7,  8,  5, 18, 16,  6, 15, 18, 14,  6, 18, 20, 20,\n",
       "        15, 18,  5,  8, 15,  9, 17,  1, 19,  4, 17,  6, 18,  9,  6, 19, 18, 15,\n",
       "        12, 15, 15,  3,  6, 15, 18,  4,  1, 18, 11,  4,  6,  4, 13,  4,  1,  8,\n",
       "        12,  4, 18,  8, 15, 15,  2, 15, 14,  6, 13, 13,  6,  1,  5,  8,  3,  9,\n",
       "         8,  3,  8,  3,  8,  4, 13,  5, 17,  6,  4,  5,  3,  3,  5, 12,  8, 18,\n",
       "        13, 17, 18,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "         0,  0])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5. example of using this on our dataset \n",
    "\n",
    "input_file = \"./fasta/acyp.fa\"\n",
    "\n",
    "train_dataset, test_dataset = create_datasets(input_file)\n",
    "\n",
    "x, y = train_dataset[0]\n",
    "\n",
    "x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. ok so to get started, let's keep good habits and record all our params in one place! \n",
    "\n",
    "@dataclass\n",
    "class ModelConfig:\n",
    "    block_size: int = None # length of the input sequences of integers\n",
    "    vocab_size: int = None # the input integers are in range [0 .. vocab_size -1]\n",
    "    n_layer: int = 4\n",
    "    embed_dim: int = 64\n",
    "    n_head: int = 4\n",
    "    learning_rate: float = 4e-3\n",
    "    batch_size: int = 32 \n",
    "    device = \"mps\"\n",
    "    max_steps = 20_000 \n",
    "\n",
    "\n",
    "config = ModelConfig() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7. define the whole transformer model\n",
    "\n",
    "class NewGELU(nn.Module):\n",
    "    \"\"\"\n",
    "    Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT).\n",
    "    Reference: Gaussian Error Linear Units (GELU) paper: https://arxiv.org/abs/1606.08415\n",
    "    \"\"\"\n",
    "\n",
    "    def forward(self, x):\n",
    "        return (0.5 * x * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)))))\n",
    "\n",
    "\n",
    "class CausalSelfAttention(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        assert config.embed_dim % config.n_head == 0\n",
    "        # key, query, value projections for all heads, but in a batch\n",
    "        self.c_attn = nn.Linear(config.embed_dim, 3 * config.embed_dim)\n",
    "        # output projection\n",
    "        self.c_proj = nn.Linear(config.embed_dim, config.embed_dim)\n",
    "        # causal mask to ensure that attention is only applied to the left in the input sequence\n",
    "        self.register_buffer(\n",
    "            \"bias\",\n",
    "            torch.tril(torch.ones(config.block_size, config.block_size)).view(\n",
    "                1, 1, config.block_size, config.block_size\n",
    "            ),\n",
    "        )\n",
    "        self.n_head = config.n_head\n",
    "        self.embed_dim = config.embed_dim\n",
    "\n",
    "    def forward(self, x):\n",
    "        B, T, C = (x.size())\n",
    "\n",
    "        # Calculate querys, keys, values for all heads in batch, using head dim as additional batch dimension \n",
    "        q, k, v = self.c_attn(x).split(self.embed_dim, dim=2)\n",
    "        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, head_size)\n",
    "        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, head_size)\n",
    "        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)  # (B, nh, T, head_size)\n",
    "\n",
    "        # Causal self-attention: (B, nh, T, hs) @ (B, nh, hs, T) -> (B, nh, T, T)\n",
    "        att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))\n",
    "        att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float(\"-inf\"))\n",
    "        att = F.softmax(att, dim=-1)\n",
    "        y = att @ v  # (B, nh, T, T) @ (B, nh, T, hs) -> (B, nh, T, hs)\n",
    "        y = (y.transpose(1, 2).contiguous().view(B, T, C))  # re-assemble all head outputs side by side\n",
    "\n",
    "        # output projection\n",
    "        y = self.c_proj(y)\n",
    "        return y\n",
    "\n",
    "\n",
    "class Block(nn.Module):\n",
    "    \"\"\"an unassuming Transformer block\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.ln_1 = nn.LayerNorm(config.embed_dim)\n",
    "        self.attn = CausalSelfAttention(config)\n",
    "        self.ln_2 = nn.LayerNorm(config.embed_dim)\n",
    "        self.mlp = nn.ModuleDict(\n",
    "            dict(\n",
    "                c_fc=nn.Linear(config.embed_dim, 4 * config.embed_dim),\n",
    "                c_proj=nn.Linear(4 * config.embed_dim, config.embed_dim),\n",
    "                act=NewGELU(),\n",
    "            )\n",
    "        )\n",
    "        m = self.mlp\n",
    "        self.mlpf = lambda x: m.c_proj(m.act(m.c_fc(x)))  # MLP forward\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.attn(self.ln_1(x))\n",
    "        x = x + self.mlpf(self.ln_2(x))\n",
    "        return x\n",
    "\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "    \"\"\"Transformer Language Model, exactly as seen in GPT-2\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.block_size = config.block_size\n",
    "\n",
    "        self.transformer = nn.ModuleDict(\n",
    "            dict(\n",
    "                wte=nn.Embedding(config.vocab_size, config.embed_dim),\n",
    "                wpe=nn.Embedding(config.block_size, config.embed_dim),\n",
    "                h=nn.ModuleList([Block(config) for _ in range(config.n_layer)]),\n",
    "                ln_f=nn.LayerNorm(config.embed_dim),\n",
    "            )\n",
    "        )\n",
    "        self.lm_head = nn.Linear(config.embed_dim, config.vocab_size, bias=False)\n",
    "\n",
    "        # report number of parameters (note we don't count the decoder parameters in lm_head)\n",
    "        n_params = sum(p.numel() for p in self.transformer.parameters())\n",
    "        print(\"number of parameters: %.2fM\" % (n_params / 1e6,))\n",
    "\n",
    "    def get_block_size(self):\n",
    "        return self.block_size\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "        device = idx.device\n",
    "        b, t = idx.size()\n",
    "        assert (\n",
    "            t <= self.block_size\n",
    "        ), f\"Cannot forward sequence of length {t}, block size is only {self.block_size}\"\n",
    "        pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0)  # shape (1, t)\n",
    "\n",
    "        # forward the GPT model itself\n",
    "        tok_emb = self.transformer.wte(idx)  # token embeddings of shape (b, t, embed_dim)\n",
    "        pos_emb = self.transformer.wpe(pos)  # position embeddings of shape (1, t, embed_dim)\n",
    "        x = tok_emb + pos_emb\n",
    "        for block in self.transformer.h:\n",
    "            x = block(x)\n",
    "        x = self.transformer.ln_f(x)\n",
    "        logits = self.lm_head(x)\n",
    "\n",
    "        # if we are given some desired targets also calculate the loss\n",
    "        loss = None\n",
    "        if targets is not None:\n",
    "            loss = F.cross_entropy(\n",
    "                logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1\n",
    "            )\n",
    "\n",
    "        return logits, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of parameters: 0.21M\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Transformer(\n",
       "  (transformer): ModuleDict(\n",
       "    (wte): Embedding(21, 64)\n",
       "    (wpe): Embedding(128, 64)\n",
       "    (h): ModuleList(\n",
       "      (0-3): 4 x Block(\n",
       "        (ln_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): CausalSelfAttention(\n",
       "          (c_attn): Linear(in_features=64, out_features=192, bias=True)\n",
       "          (c_proj): Linear(in_features=64, out_features=64, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): ModuleDict(\n",
       "          (c_fc): Linear(in_features=64, out_features=256, bias=True)\n",
       "          (c_proj): Linear(in_features=256, out_features=64, bias=True)\n",
       "          (act): NewGELU()\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=64, out_features=21, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 8. example of making a tiny model \n",
    "\n",
    "config.block_size = train_dataset.get_output_length()\n",
    "config.vocab_size = train_dataset.get_vocab_size()\n",
    "model = Transformer(config) \n",
    "\n",
    "model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0/20,000 | loss 3.2072\n",
      "step 1,000/20,000 | loss 1.4203\n",
      "step 2,000/20,000 | loss 1.5205\n",
      "step 3,000/20,000 | loss 1.2479\n",
      "step 4,000/20,000 | loss 1.5209\n",
      "step 5,000/20,000 | loss 1.3333\n",
      "step 6,000/20,000 | loss 1.2722\n",
      "step 7,000/20,000 | loss 1.3524\n",
      "step 8,000/20,000 | loss 1.3278\n",
      "step 9,000/20,000 | loss 1.3539\n",
      "step 10,000/20,000 | loss 1.1807\n",
      "step 11,000/20,000 | loss 1.2071\n",
      "step 12,000/20,000 | loss 1.3369\n",
      "step 13,000/20,000 | loss 1.2569\n",
      "step 14,000/20,000 | loss 1.3073\n",
      "step 15,000/20,000 | loss 1.3705\n",
      "step 16,000/20,000 | loss 1.2913\n",
      "step 17,000/20,000 | loss 1.1747\n",
      "step 18,000/20,000 | loss 0.9825\n",
      "step 19,000/20,000 | loss 1.4379\n"
     ]
    }
   ],
   "source": [
    "# 9. ok so now let's do a basic training loop\n",
    "\n",
    "# move model to torch device \n",
    "model.to(config.device)\n",
    "\n",
    "# init optimizer\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=config.learning_rate, betas=(0.9, 0.99), eps=1e-8)\n",
    "\n",
    "# create batch dataloader \n",
    "train_loader = DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=config.batch_size,\n",
    "    sampler=torch.utils.data.RandomSampler(train_dataset, replacement=True, num_samples=int(1e10)),\n",
    "    pin_memory=True,\n",
    ")\n",
    "\n",
    "# training loop\n",
    "step = 0\n",
    "for batch in train_loader:\n",
    "\n",
    "    # get the next batch, ship to device, and unpack it to input and target\n",
    "    batch = [t.to(config.device) for t in batch]\n",
    "    X, Y = batch\n",
    "\n",
    "    # feed into the model\n",
    "    logits, loss = model(X, Y)\n",
    "\n",
    "    # calculate the gradient, update the weights\n",
    "    model.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    # logging\n",
    "    if step % 1_000 == 0:\n",
    "        print(f\"step {step:,}/{config.max_steps:,} | loss {loss.item():.4f}\")\n",
    "\n",
    "    step += 1\n",
    "\n",
    "    # termination conditions\n",
    "    if config.max_steps >= 0 and step >= config.max_steps:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10. generate from this model \n",
    "\n",
    "@torch.no_grad()\n",
    "def generate(model, idx, max_new_tokens, temperature=1.0):\n",
    "    \"\"\"\n",
    "    Take a conditioning sequence of indices idx (LongTensor of shape (b,t)) and complete\n",
    "    the sequence max_new_tokens times, feeding the predictions back into the model each time.\n",
    "    Most likely you'll want to make sure to be in model.eval() mode of operation for this.\n",
    "    \"\"\"\n",
    "    block_size = model.get_block_size()\n",
    "    for _ in range(max_new_tokens):\n",
    "        # if the sequence context is growing too long we must crop it at block_size\n",
    "        idx_cond = idx if idx.size(1) <= block_size else idx[:, -block_size:]\n",
    "        # forward the model to get the logits for the index in the sequence\n",
    "        logits, _ = model(idx_cond)\n",
    "        # pluck the logits at the final step and scale by desired temperature\n",
    "        logits = logits[:, -1, :] / temperature\n",
    "        # apply softmax to convert logits to (normalized) probabilities\n",
    "        probs = F.softmax(logits, dim=-1)\n",
    "        # either take most likely (greedy) or sample from the distribution \n",
    "        idx_next = torch.multinomial(probs, num_samples=1)  # (B, 1)\n",
    "        # append sampled index to the running sequence and continue\n",
    "        idx = torch.cat((idx, idx_next), dim=1)  # (B, T + 1)\n",
    "\n",
    "    return idx\n",
    "\n",
    "\n",
    "def print_samples(num=10):\n",
    "    \"\"\" samples from the model and pretty prints the decoded samples \"\"\"\n",
    "    X_init = torch.zeros(num, 1, dtype=torch.long).to(config.device)\n",
    "    steps = train_dataset.get_output_length() - 1 # -1 because we already start with <START> token (index 0)\n",
    "    X_samp = generate(model, X_init, steps).to('cpu')\n",
    "    \n",
    "    samples = []\n",
    "    for i in range(X_samp.size(0)):\n",
    "        # get the i'th row of sampled integers, as python list\n",
    "        row = X_samp[i, 1:].tolist() # note: we need to crop out the first <START> token\n",
    "        # token 0 is the <STOP> token, so we crop the output sequence at that point\n",
    "        crop_index = row.index(0) if 0 in row else len(row)\n",
    "        row = row[:crop_index]\n",
    "        protein_samp = train_dataset.decode(row)\n",
    "        samples.append(protein_samp)\n",
    "        \n",
    "    for i, protein in enumerate(samples):\n",
    "        header = f\"sample_{i + 1}\"\n",
    "        print(f\">{header}\\n{protein}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">sample_1\n",
      "MAGLVSGLVSVDFEVFGKVQGVFFRKHTRKEANRLGLIGWVKNTEIGVVRMEAKQVEMQIEAVVRWLKRGPSGASNVDIEFKEIDERYEAFSVSEDFATY\n",
      ">sample_2\n",
      "MTAVYAKVRVTGRVQGVGFRYATKRGALHAAVTGWVRNDSDGSVGLLAAGAESAVDAFLAWADQGPDGASVEQVATEPESGVDDLRQIGGAFEP\n",
      ">sample_3\n",
      "MKSVFKIIYKGKVQGVGYRAFVAEIAAKDGVYGFVKNLNDGRVEILAEGENEILHMFREAIKSQFIQIIEKTEIKSIPQEVATYRKSFHTRY\n"
     ]
    }
   ],
   "source": [
    "# 11. example of generation \n",
    "\n",
    "print_samples(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 12. ok so we end with a trained model that can generate sequences, how can we tell if these are any good? \n",
    "# \n",
    "# next up, we'll build an evaluation suite for our model! "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}