GPT_Pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 准备好数据集用于训练，运行获取莎士比亚数据集\n",
    "# !wget https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据集的字符长度： 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You are all resolved rather to die than to famish?\n",
      "\n",
      "All:\n",
      "Resolved. resolved.\n",
      "\n",
      "First Citizen:\n",
      "First, you know Caius Marcius is chief enemy to the people.\n",
      "\n",
      "All:\n",
      "We know't, we know't.\n",
      "\n",
      "First Citizen:\n",
      "Let us kill him, and we'll have corn at our own price.\n",
      "Is't a verdict?\n",
      "\n",
      "All:\n",
      "No more talking on't; let it be done: away, away!\n",
      "\n",
      "Second Citizen:\n",
      "One word, good citizens.\n",
      "\n",
      "First Citizen:\n",
      "We are accounted poor\n"
     ]
    }
   ],
   "source": [
    "# 读取数据集\n",
    "with open('input.txt', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "\n",
    "print(\"数据集的字符长度：\", len(text))\n",
    "\n",
    "# 输出前500个字符内容\n",
    "print(text[:500])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
      "65\n"
     ]
    }
   ],
   "source": [
    "# 这里有出现在数据集中的全部字符\n",
    "chars = sorted(list(set(text)))\n",
    "vocab_size= len(chars)\n",
    "print(''.join(chars))\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[46, 47, 47, 1, 58, 46, 43, 56, 43]\n",
      "hii there\n"
     ]
    }
   ],
   "source": [
    "# 创建字符->整数映射\n",
    "stoi = { ch: i for i, ch in enumerate(chars) }\n",
    "itos = { i: ch for i, ch in enumerate(chars) }\n",
    "encode = lambda s: [stoi[c] for c in s] # encoder: take a string, output a list of integers\n",
    "decode = lambda l: ''.join([itos[i] for i in l]) # decoder: take a list of intgers, output a string\n",
    "\n",
    "print(encode(\"hii there\"))\n",
    "print(decode(encode(\"hii there\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1115394]) torch.int64\n",
      "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 14, 43, 44,\n",
      "        53, 56, 43,  1, 61, 43,  1, 54, 56, 53, 41, 43, 43, 42,  1, 39, 52, 63,\n",
      "         1, 44, 59, 56, 58, 46, 43, 56,  6,  1, 46, 43, 39, 56,  1, 51, 43,  1,\n",
      "        57, 54, 43, 39, 49,  8,  0,  0, 13, 50, 50, 10,  0, 31, 54, 43, 39, 49,\n",
      "         6,  1, 57, 54, 43, 39, 49,  8,  0,  0, 18, 47, 56, 57, 58,  1, 15, 47,\n",
      "        58, 47, 64, 43, 52, 10,  0, 37, 53, 59,  1, 39, 56, 43,  1, 39, 50, 50,\n",
      "         1, 56, 43, 57, 53, 50, 60, 43, 42,  1, 56, 39, 58, 46, 43, 56,  1, 58,\n",
      "        53,  1, 42, 47, 43,  1, 58, 46, 39, 52,  1, 58, 53,  1, 44, 39, 51, 47,\n",
      "        57, 46, 12,  0,  0, 13, 50, 50, 10,  0, 30, 43, 57, 53, 50, 60, 43, 42,\n",
      "         8,  1, 56, 43, 57, 53, 50, 60, 43, 42,  8,  0,  0, 18, 47, 56, 57, 58,\n",
      "         1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 18, 47, 56, 57, 58,  6,  1, 63,\n",
      "        53, 59,  1, 49, 52, 53, 61,  1, 15, 39, 47, 59, 57,  1, 25, 39, 56, 41,\n",
      "        47, 59, 57,  1, 47, 57,  1, 41, 46, 47, 43, 44,  1, 43, 52, 43, 51, 63,\n",
      "         1, 58, 53,  1, 58, 46, 43,  1, 54, 43, 53, 54, 50, 43,  8,  0,  0, 13,\n",
      "        50, 50, 10,  0, 35, 43,  1, 49, 52, 53, 61,  5, 58,  6,  1, 61, 43,  1,\n",
      "        49, 52, 53, 61,  5, 58,  8,  0,  0, 18, 47, 56, 57, 58,  1, 15, 47, 58,\n",
      "        47, 64, 43, 52, 10,  0, 24, 43, 58,  1, 59, 57,  1, 49, 47, 50, 50,  1,\n",
      "        46, 47, 51,  6,  1, 39, 52, 42,  1, 61, 43,  5, 50, 50,  1, 46, 39, 60,\n",
      "        43,  1, 41, 53, 56, 52,  1, 39, 58,  1, 53, 59, 56,  1, 53, 61, 52,  1,\n",
      "        54, 56, 47, 41, 43,  8,  0, 21, 57,  5, 58,  1, 39,  1, 60, 43, 56, 42,\n",
      "        47, 41, 58, 12,  0,  0, 13, 50, 50, 10,  0, 26, 53,  1, 51, 53, 56, 43,\n",
      "         1, 58, 39, 50, 49, 47, 52, 45,  1, 53, 52,  5, 58, 11,  1, 50, 43, 58,\n",
      "         1, 47, 58,  1, 40, 43,  1, 42, 53, 52, 43, 10,  1, 39, 61, 39, 63,  6,\n",
      "         1, 39, 61, 39, 63,  2,  0,  0, 31, 43, 41, 53, 52, 42,  1, 15, 47, 58,\n",
      "        47, 64, 43, 52, 10,  0, 27, 52, 43,  1, 61, 53, 56, 42,  6,  1, 45, 53,\n",
      "        53, 42,  1, 41, 47, 58, 47, 64, 43, 52, 57,  8,  0,  0, 18, 47, 56, 57,\n",
      "        58,  1, 15, 47, 58, 47, 64, 43, 52, 10,  0, 35, 43,  1, 39, 56, 43,  1,\n",
      "        39, 41, 41, 53, 59, 52, 58, 43, 42,  1, 54, 53, 53, 56])\n"
     ]
    }
   ],
   "source": [
    "# 编码整个文本数据集并且将它储存在torch.Tensor中\n",
    "import torch\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "print(data.shape, data.dtype)\n",
    "print(data[:500]) # 我们上面刚看到的500个字符对于GPT来说看到的是下面这样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据集切分为训练集和验证集\n",
    "n = int(0.9*len(data)) # 90%用于训练，剩余10%用于验证\n",
    "train_data = data[:n]\n",
    "val_data = data[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58])\n",
      "when input is tensor([18]) the target: 47\n",
      "when input is tensor([18, 47]) the target: 56\n",
      "when input is tensor([18, 47, 56]) the target: 57\n",
      "when input is tensor([18, 47, 56, 57]) the target: 58\n",
      "when input is tensor([18, 47, 56, 57, 58]) the target: 1\n",
      "when input is tensor([18, 47, 56, 57, 58,  1]) the target: 15\n",
      "when input is tensor([18, 47, 56, 57, 58,  1, 15]) the target: 47\n",
      "when input is tensor([18, 47, 56, 57, 58,  1, 15, 47]) the target: 58\n"
     ]
    }
   ],
   "source": [
    "block_size = 8\n",
    "print(train_data[:block_size+1])\n",
    "\n",
    "x = train_data[:block_size]\n",
    "y = train_data[1:block_size+1]\n",
    "\n",
    "for t in range(block_size):\n",
    "    context = x[:t+1]\n",
    "    target = y[t]\n",
    "    print(f\"when input is {context} the target: {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入：\n",
      "torch.Size([4, 8])\n",
      "tensor([[58, 46, 39, 58,  1, 57, 53,  1],\n",
      "        [12,  1,  1, 51, 63,  1, 52, 47],\n",
      "        [46, 44, 59, 50,  1, 57, 53, 59],\n",
      "        [43, 50, 57, 43,  0, 32, 53,  1]])\n",
      "目标：\n",
      "torch.Size([4, 8])\n",
      "tensor([[46, 39, 58,  1, 57, 53,  1, 50],\n",
      "        [ 1,  1, 51, 63,  1, 52, 47, 43],\n",
      "        [44, 59, 50,  1, 57, 53, 59, 50],\n",
      "        [50, 57, 43,  0, 32, 53,  1, 46]])\n",
      "--------------------\n",
      "when input is [58] the target: 46\n",
      "when input is [58, 46] the target: 39\n",
      "when input is [58, 46, 39] the target: 58\n",
      "when input is [58, 46, 39, 58] the target: 1\n",
      "when input is [58, 46, 39, 58, 1] the target: 57\n",
      "when input is [58, 46, 39, 58, 1, 57] the target: 53\n",
      "when input is [58, 46, 39, 58, 1, 57, 53] the target: 1\n",
      "when input is [58, 46, 39, 58, 1, 57, 53, 1] the target: 50\n",
      "when input is [12] the target: 1\n",
      "when input is [12, 1] the target: 1\n",
      "when input is [12, 1, 1] the target: 51\n",
      "when input is [12, 1, 1, 51] the target: 63\n",
      "when input is [12, 1, 1, 51, 63] the target: 1\n",
      "when input is [12, 1, 1, 51, 63, 1] the target: 52\n",
      "when input is [12, 1, 1, 51, 63, 1, 52] the target: 47\n",
      "when input is [12, 1, 1, 51, 63, 1, 52, 47] the target: 43\n",
      "when input is [46] the target: 44\n",
      "when input is [46, 44] the target: 59\n",
      "when input is [46, 44, 59] the target: 50\n",
      "when input is [46, 44, 59, 50] the target: 1\n",
      "when input is [46, 44, 59, 50, 1] the target: 57\n",
      "when input is [46, 44, 59, 50, 1, 57] the target: 53\n",
      "when input is [46, 44, 59, 50, 1, 57, 53] the target: 59\n",
      "when input is [46, 44, 59, 50, 1, 57, 53, 59] the target: 50\n",
      "when input is [43] the target: 50\n",
      "when input is [43, 50] the target: 57\n",
      "when input is [43, 50, 57] the target: 43\n",
      "when input is [43, 50, 57, 43] the target: 0\n",
      "when input is [43, 50, 57, 43, 0] the target: 32\n",
      "when input is [43, 50, 57, 43, 0, 32] the target: 53\n",
      "when input is [43, 50, 57, 43, 0, 32, 53] the target: 1\n",
      "when input is [43, 50, 57, 43, 0, 32, 53, 1] the target: 46\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(1024)\n",
    "batch_size = 4 # 含义为多少独立序列会被并行处理\n",
    "block_size = 8 # 含义为预测的最大内容长度\n",
    "\n",
    "def get_batch(split):\n",
    "    # 生成一个小批量数据输入x和目标y的数据\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    return x, y\n",
    "\n",
    "xb, yb = get_batch('train')\n",
    "print(\"输入：\")\n",
    "print(xb.shape)\n",
    "print(xb)\n",
    "print(\"目标：\")\n",
    "print(yb.shape)\n",
    "print(yb)\n",
    "\n",
    "print(\"-\"*20)\n",
    "\n",
    "for b in range(batch_size): # batch维度\n",
    "    for t in range(block_size): # 时间维度\n",
    "        context = xb[b, :t+1]\n",
    "        target = yb[b, t]\n",
    "        print(f\"when input is {context.tolist()} the target: {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[58, 46, 39, 58,  1, 57, 53,  1],\n",
      "        [12,  1,  1, 51, 63,  1, 52, 47],\n",
      "        [46, 44, 59, 50,  1, 57, 53, 59],\n",
      "        [43, 50, 57, 43,  0, 32, 53,  1]]) tensor([[46, 39, 58,  1, 57, 53,  1, 50],\n",
      "        [ 1,  1, 51, 63,  1, 52, 47, 43],\n",
      "        [44, 59, 50,  1, 57, 53, 59, 50],\n",
      "        [50, 57, 43,  0, 32, 53,  1, 46]])\n"
     ]
    }
   ],
   "source": [
    "print(xb, yb) # our input to the transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "torch.manual_seed(1024)\n",
    "\n",
    "class BigramLanguageModel(nn.Module):\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        # 每个token直接从查找表中读取下一个token的logits\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "\n",
    "    def forward(self, idx, targets=None):\n",
    "        # idx和targets都是 (B, T)维度的数字tensor\n",
    "        logits = self.token_embedding_table(idx) # (B,T,C)\n",
    "        \n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        # idx是当前上下文中的索引数组，形状为(B,T)\n",
    "        for _ in range(max_new_tokens):\n",
    "            # 获取预测值\n",
    "            logits, loss = self(idx) # 注意此处self代表BigramLanguageModel类\n",
    "            # 只集中在最后时间步上\n",
    "            logits = logits[:, -1, :] # (B,C)\n",
    "            # 应用softmax去获取概率值\n",
    "            probs = F.softmax(logits, dim=-1) # (B,C)\n",
    "            # 从分布中抽样\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "            # 添加抽样index到预测序列中\n",
    "            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)\n",
    "        return idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 65]) tensor(4.7921, grad_fn=<NllLossBackward0>)\n",
      "\n",
      "YpD&;JVTO,KBk:nicwp\n",
      "P-wXPZEtUgtAgM?i3oUU?:'zTokrAVxhireSkj-cjhaqEw:baA,:z.ohoayokMFHHWMf;ujJXoM?HmYk\n"
     ]
    }
   ],
   "source": [
    "m = BigramLanguageModel(vocab_size)\n",
    "logits, loss = m(xb, yb)\n",
    "print(logits.shape, loss)\n",
    "\n",
    "print(decode(m.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=100)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.139368772506714\n",
      "\n",
      "'BYoull,?vxNZfout,WS.WRinblre FRSThe,3IENGLUClsCut'e JdDUvSghyQM.f aTeryreo\n",
      "LdYXCBfousTYSbovuxxMISjvuFaioUKESMAOyaxjA!wH3l$?jrows.ntqfRanLd\n",
      "Dmy,ci, CpJ:.ns IfRallFLorBure,\n",
      "Ge\n",
      "Bju oud:SPer\n",
      "TonYYworaxY'of ty.RXMtJEPEJXTMIEmy?\n",
      "A\n",
      "B'?ZVwouL$oure n fjjPce,G-c!\n",
      "U&;Ke;OqAcXc onmaTADihe Wq-oMANo? pollPlmYUNCAs l enFLiF&w'd'sicheryo bRO:ZlP.LICAg.WqYEu lavYmn.V;k, ypurPqkNgempeCKHsWsurias:\n",
      ".rr WrorcaXTererp;!sm O:\n",
      "S&ot o b nxzLIhakiBjhe.M.zmNUUUye,;Kxcl;NerotsinkeUgiqY'Pags t'e !XLK'sty I:ESAR:LZFRe wincj\n"
     ]
    }
   ],
   "source": [
    "# 创建一个Pytorh优化器\n",
    "optimizer = torch.optim.AdamW(m.parameters(), lr=1e-3)\n",
    "\n",
    "batch_size = 32\n",
    "for steps in range(100): # 增加step数来获得更好结果\n",
    "    # 抽样一批数据\n",
    "    xb, yb = get_batch(\"train\")\n",
    "\n",
    "    # 评估loss\n",
    "    logits, loss = m(xb, yb)\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "print(loss.item()) # 多刷新几次会发现loss单调下降（梯度生效）\n",
    "print(decode(m.generate(idx=torch.zeros((1, 1), dtype=torch.long), max_new_tokens=500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a =  tensor([[1., 0., 0.],\n",
      "        [1., 1., 0.],\n",
      "        [1., 1., 1.]])\n",
      "a ->  tensor([[1.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333]])\n",
      "b =  tensor([[2., 7.],\n",
      "        [6., 4.],\n",
      "        [6., 5.]])\n",
      "c =  tensor([[2.0000, 7.0000],\n",
      "        [4.0000, 5.5000],\n",
      "        [4.6667, 5.3333]])\n",
      "x.shape =  torch.Size([4, 8, 2])\n",
      "xbow.shape =  torch.Size([4, 8, 2])\n"
     ]
    }
   ],
   "source": [
    "# self-attention中的数学trick\n",
    "torch.manual_seed(42)\n",
    "a = torch.tril(torch.ones(3, 3))\n",
    "print(\"a = \", a)\n",
    "a = a / torch.sum(a, 1, keepdim=True)\n",
    "print(\"a -> \", a)\n",
    "b = torch.randint(0, 10, (3, 2)).float()\n",
    "print(\"b = \", b)\n",
    "c = a @ b\n",
    "print(\"c = \", c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x.shape =  torch.Size([4, 8, 2])\n",
      "方案二输出是否等于方案一输出： True\n",
      "方案三输出是否等于方案一输出： True\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 16])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 进一步扩展上述例子\n",
    "torch.manual_seed(1024)\n",
    "B, T, C = 4, 8, 2 # batch, time, channels\n",
    "x = torch.randn(B, T, C)\n",
    "print(\"x.shape = \", x.shape)\n",
    "# 方案一：使用for训练\n",
    "# x[b, t] = mean_{i<=t} x[b,i]\n",
    "xbow = torch.zeros((B, T, C)) # bow = bag of words\n",
    "for b in range(B): \n",
    "    for t in range(T):\n",
    "        xprev = x[b, :t+1]\n",
    "        xbow[b, t] = torch.mean(xprev, 0)\n",
    "# print(\"xbow.shape = \", xbow.shape)\n",
    "# 方案二：使用矩阵乘积求加权聚合\n",
    "wei = torch.tril(torch.ones(T, T)) # (T, T)\n",
    "wei = wei / wei.sum(1, keepdim=True)\n",
    "xbow2 = wei @ x # (B, T, T) @ (B, T, C) --> (B, T, C)\n",
    "print(\"方案二输出是否等于方案一输出：\", torch.allclose(xbow, xbow2))\n",
    "# 方案三：使用softmax\n",
    "tril = torch.tril(torch.ones(T, T))\n",
    "wei = torch.zeros((T,T))\n",
    "wei = wei.masked_fill(tril == 0, float('-inf')) # 使用masked_fill\n",
    "wei = F.softmax(wei, dim=-1)\n",
    "xbow3 = wei @ x\n",
    "print(\"方案三输出是否等于方案一输出：\", torch.allclose(xbow, xbow3))\n",
    "# 方案四：self-attention（单头）\n",
    "head_size = 16\n",
    "key = nn.Linear(C, head_size, bias=False)\n",
    "query = nn.Linear(C, head_size, bias=False)\n",
    "value = nn.Linear(C, head_size, bias=False)\n",
    "k = key(x)   # (B, T, 16)\n",
    "q = query(x) # (B, T, 16)\n",
    "wei =  q @ k.transpose(-2, -1) # (B, T, 16) @ (B, 16, T) ---> (B, T, T)\n",
    "\n",
    "tril = torch.tril(torch.ones(T, T))\n",
    "wei = wei.masked_fill(tril == 0, float('-inf'))\n",
    "wei = F.softmax(wei, dim=-1)\n",
    "v = value(x)\n",
    "out = wei @ v\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Atetntion`是一种`沟通机制`。可以看作是有向图中的节点，它们相互观察，并通过指向它们的所有节点的加权和(具有数据依赖的权重)聚合信息。\n",
    "- 没有空间的概念。注意力只是作用于一组向量。这就是为什么我们需要对令牌进行位置编码。\n",
    "- 跨批处理维度的每个示例当然是完全独立处理的，并且永远不会相互“交谈”\n",
    "- 在`encoder`注意块中，只需删除用`tril`屏蔽的单行，允许所有令牌进行通信。这里的这个块被称为`decoder`注意力块，因为它有三角形掩蔽，通常用于自回归设置，比如语言建模。\n",
    "- `self-attention`只是意味着键和值与查询来自相同的来源。在“交叉关注”中，查询仍然由x产生，但键和值来自其他外部源(例如编码器模块)。\n",
    "- `scaled attention`额外将`wei`除以1/sqrt(head_size)。这使得当输入`Q`,`K`是单位方差时，`wei`也将是单位方差，`Softmax`将保持扩散而不是过度饱和"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 100])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LayerNorm，可参考Transformer中更为系统的实现：https://github.com/PhenixZhang/Transformer-Pytorch\n",
    "class LayerNorm1d: \n",
    "\n",
    "  def __init__(self, dim, eps=1e-5, momentum=0.1):\n",
    "    self.eps = eps\n",
    "    self.gamma = torch.ones(dim)\n",
    "    self.beta = torch.zeros(dim)\n",
    "\n",
    "  def __call__(self, x):\n",
    "    xmean = x.mean(1, keepdim=True) # batch均值\n",
    "    xvar = x.var(1, keepdim=True) # batch方差\n",
    "    xhat = (x - xmean) / torch.sqrt(xvar + self.eps) # 归一化\n",
    "    self.out = self.gamma * xhat + self.beta\n",
    "    return self.out\n",
    "\n",
    "  def parameters(self):\n",
    "    return [self.gamma, self.beta]\n",
    "\n",
    "torch.manual_seed(1337)\n",
    "module = LayerNorm1d(100)\n",
    "x = torch.randn(32, 100) # batch_size32，100维embedding\n",
    "x = module(x)\n",
    "x.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}