PyTorch_bigram_Eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "## Note , this is a CPU based MAC System and if you are using  APPLE M3 MAC with GPU, then you can directly run in GPU  Hardware.\n",
    "print(device)\n",
    "block_size = 8\n",
    "batch_size = 4\n",
    "max_iters = 1000\n",
    "# eval_interval = 2500\n",
    "learning_rate = 3e-4\n",
    "eval_iters = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n', ' ', '!', '\"', '&', \"'\", '(', ')', '*', ',', '-', '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']', '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '\\ufeff']\n"
     ]
    }
   ],
   "source": [
    "with open('wizard_of_oz.txt', 'r', encoding='utf-8') as f:\n",
    "    text = f.read()\n",
    "chars = sorted(set(text))\n",
    "print(chars)\n",
    "vocab_size = len(chars)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([80,  1,  1, 28, 39, 42, 39, 44, 32, 49,  1, 25, 38, 28,  1, 44, 32, 29,\n",
      "         1, 47, 33, 50, 25, 42, 28,  1, 33, 38,  1, 39, 50,  0,  0,  1,  1, 26,\n",
      "        49,  0,  0,  1,  1, 36, 11,  1, 30, 42, 25, 38, 35,  1, 26, 25, 45, 37,\n",
      "         0,  0,  1,  1, 25, 45, 44, 32, 39, 42,  1, 39, 30,  1, 44, 32, 29,  1,\n",
      "        47, 33, 50, 25, 42, 28,  1, 39, 30,  1, 39, 50,  9,  1, 44, 32, 29,  1,\n",
      "        36, 25, 38, 28,  1, 39, 30,  1, 39, 50,  9,  1, 39, 50, 37, 25,  1, 39,\n",
      "        30,  1, 39, 50,  9,  1, 29, 44, 27, 11,  0,  0,  1,  1, 33, 36, 36, 45,\n",
      "        43, 44, 42, 25, 44, 29, 28,  1, 26, 49,  1, 34, 39, 32, 38,  1, 42, 11,\n",
      "         1, 38, 29, 33, 36, 36,  0,  0,  1,  1, 26, 39, 39, 35, 43,  1, 39, 30,\n",
      "         1, 47, 39, 38, 28, 29, 42,  1, 47, 33, 36, 36, 33, 25, 37,  1, 37, 39,\n",
      "        42, 42, 39, 47,  1,  4,  1, 27, 39, 11,  9,  1, 33, 38, 27, 11,  1, 38,\n",
      "        29, 47])\n"
     ]
    }
   ],
   "source": [
    "string_to_int = { ch:i for i,ch in enumerate(chars) }\n",
    "int_to_string = { i:ch for i,ch in enumerate(chars) }\n",
    "encode = lambda s: [string_to_int[c] for c in s]\n",
    "decode = lambda l: ''.join([int_to_string[i] for i in l])\n",
    "\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "print(data[:200])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "185847\n",
      "232309\n",
      "inputs:\n",
      "tensor([[61, 58,  1, 71, 68, 68, 59, 24],\n",
      "        [57, 57, 62, 67, 60,  1, 54, 73],\n",
      "        [62, 79, 54, 71, 57,  9,  1, 76],\n",
      "        [72,  1, 59, 68, 71, 66, 58, 71]])\n",
      "targets:\n",
      "tensor([[58,  1, 71, 68, 68, 59, 24,  3],\n",
      "        [57, 62, 67, 60,  1, 54, 73,  1],\n",
      "        [79, 54, 71, 57,  9,  1, 76, 61],\n",
      "        [ 1, 59, 68, 71, 66, 58, 71,  1]])\n"
     ]
    }
   ],
   "source": [
    "n = int(0.8*len(data))\n",
    "print(n)\n",
    "print(len(data))\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]\n",
    "\n",
    "def get_batch(split):\n",
    "    data = train_data if split == 'train' else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    x, y = x.to(device), y.to(device)\n",
    "    return x, y\n",
    "\n",
    "x, y = get_batch('train')\n",
    "print('inputs:')\n",
    "# print(x.shape)\n",
    "print(x)\n",
    "print('targets:')\n",
    "print(y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def estimate_loss():\n",
    "    out = {}\n",
    "    model.eval()\n",
    "    for split in ['train', 'val']:\n",
    "        losses = torch.zeros(eval_iters)\n",
    "        for k in range(eval_iters):\n",
    "            X, Y = get_batch(split)\n",
    "            logits, loss = model(X, Y)\n",
    "            losses[k] = loss.item()\n",
    "        out[split] = losses.mean()\n",
    "    model.train()\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "n\"ksMu-hU],66kc﻿*ouNC_TCirP\"d5qzxjA3rz25RQxyhXJ3OCA ;]466rcZ697cczsUFD92p6'xl2bHkxMOGFHq\"uypXNwQY1_jAi6HlA-U3OLfU(E5oVZDQgZlxnqREW1j:6!9MLv;U'SsKU6&w?,3n:oBL:e6 h(HEQkn?*Q]yAbp(h4)&zbpJ\"CwCGQh,v'9XHe4!G0 xX\"pzwb5e]PJn \"53DrDPP\n",
      "29\"'[&e4:oV[﻿Q [UCE3s]JKs\n",
      "mKq[!m3 czxyMr6gVuTCF:8p1h]q[ Bok(\n",
      "BYS7JP:W1h(3E)kn.dbmKBiJ)wBpafWw,0BS?,l)1nd3[5l'JRyE?Wr)VyALtW*!\"gn.z8S3rex8p9')9Q2;]X'2O_\"4YNCc8&[5k(8OFO_QKd2[.H*1(gVh,vEy6g(G.c\"&zKloVZf35ea5E2Zv;Kl4!j'a.xnssGB\n",
      "qJ!eY-lU]v'OG\"8Bom\n",
      "?t:!D5qeglL;n3vW1'ToXU1﻿[:r*f\n"
     ]
    }
   ],
   "source": [
    "class BigramLanguageModel(nn.Module):\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "        \n",
    "    def forward(self, index, targets=None):\n",
    "        logits = self.token_embedding_table(index)\n",
    "        \n",
    "        \n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C)\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "        \n",
    "        return logits, loss\n",
    "    \n",
    "    def generate(self, index, max_new_tokens):\n",
    "        # index is (B, T) array of indices in the current context\n",
    "        for _ in range(max_new_tokens):\n",
    "            # get the predictions\n",
    "            logits, loss = self.forward(index)\n",
    "            # focus only on the last time step\n",
    "            logits = logits[:, -1, :] # becomes (B, C)\n",
    "            # apply softmax to get probabilities\n",
    "            probs = F.softmax(logits, dim=-1) # (B, C)\n",
    "            # sample from the distribution\n",
    "            index_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "            # append sampled index to the running sequence\n",
    "            index = torch.cat((index, index_next), dim=1) # (B, T+1)\n",
    "        return index\n",
    "\n",
    "model = BigramLanguageModel(vocab_size)\n",
    "m = model.to(device)\n",
    "\n",
    "context = torch.zeros((1,1), dtype=torch.long, device=device)\n",
    "generated_chars = decode(m.generate(context, max_new_tokens=500)[0].tolist())\n",
    "print(generated_chars)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 0, train loss: 4.790, val loss: 4.819\n",
      "step: 250, train loss: 4.732, val loss: 4.738\n",
      "step: 500, train loss: 4.668, val loss: 4.677\n",
      "step: 750, train loss: 4.614, val loss: 4.623\n",
      "4.56766939163208\n"
     ]
    }
   ],
   "source": [
    "# create a PyTorch optimizer\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for iter in range(max_iters):\n",
    "    if iter % eval_iters == 0:\n",
    "        losses = estimate_loss()\n",
    "        print(f\"step: {iter}, train loss: {losses['train']:.3f}, val loss: {losses['val']:.3f}\")\n",
    "\n",
    "    # sample a batch of data\n",
    "    xb, yb = get_batch('train')\n",
    "\n",
    "    # evaluate the loss\n",
    "    logits, loss = model.forward(xb, yb)\n",
    "    optimizer.zero_grad(set_to_none=True)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***need to familiarize audience with optimizers (AdamW, Adam, SGD, MSE…) no need to jump into the formulas, just what the optimizer does for us and some of the differences/similarities between them***\n",
    "\n",
    "1. **Mean Squared Error (MSE)**: MSE is a common loss function used in regression problems, where the goal is to predict a continuous output. It measures the average squared difference between the predicted and actual values, and is often used to train neural networks for regression tasks.\n",
    "2. **Gradient Descent (GD):**  is an optimization algorithm used to minimize the loss function of a machine learning model. The loss function measures how well the model is able to predict the target variable based on the input features. The idea of GD is to iteratively adjust the model parameters in the direction of the steepest descent of the loss function\n",
    "3. **Momentum**: Momentum is an extension of SGD that adds a \"momentum\" term to the parameter updates. This term helps smooth out the updates and allows the optimizer to continue moving in the right direction, even if the gradient changes direction or varies in magnitude. Momentum is particularly useful for training deep neural networks.\n",
    "4. **RMSprop**: RMSprop is an optimization algorithm that uses a moving average of the squared gradient to adapt the learning rate of each parameter. This helps to avoid oscillations in the parameter updates and can improve convergence in some cases.\n",
    "5. **Adam**: Adam is a popular optimization algorithm that combines the ideas of momentum and RMSprop. It uses a moving average of both the gradient and its squared value to adapt the learning rate of each parameter. Adam is often used as a default optimizer for deep learning models.\n",
    "6. **AdamW**: AdamW is a modification of the Adam optimizer that adds weight decay to the parameter updates. This helps to regularize the model and can improve generalization performance. We will be using the AdamW optimizer as it best suits the properties of the model we will train in this video.\n",
    "\n",
    "find more optimizers and details at torch.optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "W*f-3F4d7Vxys RA*Y1q6﻿2w,E:so!XvNwfi0t.AML'QY7eee2Mu Y[K)﻿V_aTfn \"56T78'h,A7]XU:(dqm﻿kA﻿8ZcwAM&bS_TA Yn3DXq)a!YB\n",
      "LZ8py0.Xcu4bUAKob9b hfo NE\"0X;t'9:8(2dq[W:ZH0!Qn-t[W[LD?Ea?7﻿,09t)e)w;CFX.6YsKz\n",
      "3i6.W;Tv3omA:T[aD.sKgi4GX\n",
      "yqAIMLj'7Yfa*]c2!8(e8FBR.a*xN\n",
      "kbvRRcz(Ofdjlnzc\n",
      "CJ!l-oWTFVh(E&nFCFO;(3OVrc9﻿'[622oXyYOJRt&UCFV- XUvx;thjc8']dPldj[)C(BoV']Bt!wX\"9W*(EQ\"mbIO;V7&gf_aD4\"ivg( hfRohQ74Q*mb]*;3n4:f7fwYTOt],Pla3﻿a*&OZ[5IROOwhl*6b*Z(WE[WeK L,vNBXVPl*dHBUOCiHn3-e \"doO;PLjAmO_:uloJSLRlLP[QKd?p.D\" cWDb]yzYN9\n"
     ]
    }
   ],
   "source": [
    "'''context = torch.zeros((1,1), dtype=torch.long, device=device)\n",
    "gen_values = m.generate(context, max_new_tokens=500)\n",
    "                        \n",
    "print(f\"\\n GEN_VALUES:: {gen_values}\") \n",
    "#generated_chars = decode(gen)_values[0].tolist())\n",
    "print(generated_chars)'''\n",
    "\n",
    "context = torch.zeros((1,1), dtype=torch.long, device=device)\n",
    "generated_chars = decode(m.generate(context, max_new_tokens=500)[0].tolist())\n",
    "print(generated_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}