lecture_04.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lecture 04:\n",
    "\n",
    "**Backpropagation**: Algorithm to caculate gradient  for all the weights in the network with several weights. \n",
    "\n",
    "* It uses the `Chain Rule` to calcuate the gradient for multiple nodes at the same time. \n",
    "* In pytorch this is implemented using a `variable` data type and `loss.backward()` method to get the gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using .backward method for linear regression\n",
    "\n",
    "$y = x * w$\n",
    "\n",
    "$loss =(\\hat{y}-y)^2$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Input data\n",
    "\n",
    "x_data = [1.0, 2.0, 3.0]\n",
    "y_data = [2.0, 4.0, 6.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = torch.tensor([1.0], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(x):\n",
    "    return x*w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(y_pred,y):\n",
    "    return (y_pred-y)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict (before training) 4 4.0\n",
      "\tgrad:  1.0 2.0 -2.0\n",
      "\tgrad:  2.0 4.0 -7.840000152587891\n",
      "\tgrad:  3.0 6.0 -16.228801727294922\n",
      "Epoch: 0 | Loss: 7.315943717956543 | w: 1.260688066482544\n",
      "\tgrad:  1.0 2.0 -1.478623867034912\n",
      "\tgrad:  2.0 4.0 -5.796205520629883\n",
      "\tgrad:  3.0 6.0 -11.998146057128906\n",
      "Epoch: 1 | Loss: 3.9987640380859375 | w: 1.4534177780151367\n",
      "\tgrad:  1.0 2.0 -1.0931644439697266\n",
      "\tgrad:  2.0 4.0 -4.285204887390137\n",
      "\tgrad:  3.0 6.0 -8.870372772216797\n",
      "Epoch: 2 | Loss: 2.1856532096862793 | w: 1.5959051847457886\n",
      "\tgrad:  1.0 2.0 -0.8081896305084229\n",
      "\tgrad:  2.0 4.0 -3.1681032180786133\n",
      "\tgrad:  3.0 6.0 -6.557973861694336\n",
      "Epoch: 3 | Loss: 1.1946394443511963 | w: 1.7012479305267334\n",
      "\tgrad:  1.0 2.0 -0.5975041389465332\n",
      "\tgrad:  2.0 4.0 -2.3422164916992188\n",
      "\tgrad:  3.0 6.0 -4.848389625549316\n",
      "Epoch: 4 | Loss: 0.6529689431190491 | w: 1.779128909111023\n",
      "\tgrad:  1.0 2.0 -0.4417421817779541\n",
      "\tgrad:  2.0 4.0 -1.7316293716430664\n",
      "\tgrad:  3.0 6.0 -3.58447265625\n",
      "Epoch: 5 | Loss: 0.35690122842788696 | w: 1.836707353591919\n",
      "\tgrad:  1.0 2.0 -0.3265852928161621\n",
      "\tgrad:  2.0 4.0 -1.2802143096923828\n",
      "\tgrad:  3.0 6.0 -2.650045394897461\n",
      "Epoch: 6 | Loss: 0.195076122879982 | w: 1.8792757987976074\n",
      "\tgrad:  1.0 2.0 -0.24144840240478516\n",
      "\tgrad:  2.0 4.0 -0.9464778900146484\n",
      "\tgrad:  3.0 6.0 -1.9592113494873047\n",
      "Epoch: 7 | Loss: 0.10662525147199631 | w: 1.9107471704483032\n",
      "\tgrad:  1.0 2.0 -0.17850565910339355\n",
      "\tgrad:  2.0 4.0 -0.699742317199707\n",
      "\tgrad:  3.0 6.0 -1.4484672546386719\n",
      "Epoch: 8 | Loss: 0.0582793727517128 | w: 1.9340143203735352\n",
      "\tgrad:  1.0 2.0 -0.1319713592529297\n",
      "\tgrad:  2.0 4.0 -0.5173273086547852\n",
      "\tgrad:  3.0 6.0 -1.070866584777832\n",
      "Epoch: 9 | Loss: 0.03185431286692619 | w: 1.9512161016464233\n",
      "Predict (After training) 4 hours 7.804864406585693\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "\n",
    "print('Predict (before training)', 4, forward(4).item())\n",
    "\n",
    "# Training loop\n",
    "\n",
    "for epoch in range(10):\n",
    "#     l_sum=0\n",
    "    for x_val, y_val in zip(x_data, y_data):\n",
    "        y_pred = forward(x_val) # Forward pass\n",
    "        l = loss(y_pred,y_val) # Loss\n",
    "        l.backward() # Backpropagation\n",
    "        print(\"\\tgrad: \", x_val, y_val, w.grad.item())\n",
    "        w.data = w.data - 0.01 * w.grad.item()\n",
    "\n",
    "        # Manually zero the gradients after updating weights\n",
    "        w.grad.data.zero_()\n",
    "        \n",
    "    print(f\"Epoch: {epoch} | Loss: {l.item()} | w: {w.item()}\")\n",
    "#     w_list.append(w)\n",
    "#     mse_list.append(l_sum/3)\n",
    "    \n",
    "    \n",
    "print('Predict (After training)', '4 hours', forward(4).item())    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using backward propagation for quadratic model\n",
    "\n",
    "$\\hat{y} = x^2*w_{2} + x*w_{1} + b$\n",
    "\n",
    "$loss = (\\hat{y}-y)^2$\n",
    "\n",
    "* Using Dummy values of x and y\n",
    "\n",
    "`x = 1,2,3,4,5`\n",
    "`y = 1,6,15,28,45`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_data = [1.0, 2.0, 3.0, 4.0, l5.0]\n",
    "y_data = [1.0, 6.0, 15.0, 28,45]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAfIUlEQVR4nO3dd3hUZcL+8e9DIPRqEkACBqSEIiUERFHEoIIVKyLFsiqrYl/L6u777ru6TVzbrq6ugm0FBdtaV1TK2lYghCYkoYYigSQEAklImZnn98cM/lgIMCEzc+ZM7s91cTHlhHPzkLl5MnPOeYy1FhERcZ8GTgcQEZHjowIXEXEpFbiIiEupwEVEXEoFLiLiUg0jubOEhASbkpISyV2KiLje0qVLi6y1iYc+HtECT0lJITMzM5K7FBFxPWPM5poe11soIiIupQIXEXEpFbiIiEupwEVEXEoFLiLiUipwERGXUoGLiLiUClxEJIyKSit55KM1VHq8If+zVeAiImFSVFrJhJe+Z9bizazbWRryP18FLiISBrtKK5n40iK2FJfz8vVD6Nepdcj3oQIXEQmxXaWVTHhpEZuLy3j5uiGcfnJCWPajAhcRCaFdpZVMnL6IvF1lzLhuCKd3D095gwpcRCRkisuqmDh9EZuKynj5+iEMD2N5gwpcRCQkisuqmPDS92wq8s+8w13eoAIXEamz3QfNvKdfl84ZPcJf3hDh64GLiMSa3WVVTJi+iA2FpUy/Np0zexy27kLYaAYuInKcDsy8D5T3iJ6RK29QgYuIHJc95VVMmrGI9YWlvORAeYMKXESk1vaU+2fe6wpKeXHyYM5yoLxBBS4iUisHZt7rdvrLe2SvJMeyqMBFRIJUUl7NpBmLWLujlL9f62x5gwpcRCQoJeXVTJzxvb+8Jw/mbIfLG1TgIiLHdPDM+4XJaZyd6nx5gwpcROSoSvZXM/nlReTu2Mfzk9LISG3vdKSfqMBFRI6gZH81k2csIjt/L89PSmNU7+gpb1CBi4jUqGR/NdcGyvuFSYOjrrxBBS4icpi9FdVc+/Ji1uTv5fmJ0VneoAIXEfkveyuqmTxjMWu2l/C3iYM5p090ljfUosCNMXHGmGXGmI8D97saYxYZY9YZY2YbY+LDF1NEJPz2VlRzbaC8n5uQxrlRXN5Quxn4XUD2QfcfA56y1vYAdgM3hjKYiEgk7auo5rqXF/PDj/7yPq9vB6cjHVNQBW6MSQYuBKYH7hsgA3gnsMlrwKXhCCgiEm77Au95r9pWwnMT3VHeEPwM/GngAcAXuH8CsMda6wnc3wZ0qukLjTFTjDGZxpjMwsLCOoUVEQm1AzPvVdtKeHZCGqNdUt4QRIEbYy4CCqy1Sw9+uIZNbU1fb6190Vqbbq1NT0x05opdIiI1Ka30cP0rS1i5rYRnJwxiTD/3lDcEtyLPcOASY8wFQBOgFf4ZeRtjTMPALDwZ2B6+mCIioVVa6eG6lxezfOsenr1mEGP6dXQ6Uq0dcwZurX3IWptsrU0BxgPzrbUTgQXAlYHNrgM+CFtKEZEQKq30cP1B5X3+Ke4rb6jbceAPAvcaY9bjf098RmgiiYiET2mlhxteWcyyrXv4q4vLG2q5qLG1diGwMHB7IzA09JFERMLjQHlnbdnDX8YP4gIXlzfoTEwRqSfKDirvZ8YP5ML+7i5vUIGLSD3gL+8lZG3Zw9NXD+Si/ic6HSkkVOAiEtPKKj3c8OoSMjcX8/TVA7l4QGyUN6jARSSGlVcFyjuvmKfHD4qp8gYVuIjEqPIq/9smmXnFPHX1QC6JsfIGFbiIxKDyKg8/e3UJSwLlPXZgjVf6cD0VuIjElP1VXm58NZPFm2K7vEEFLiIxZH+Vl5+9uoRFm3bx5LjYLm9QgYtIjNhf5eXG1/zl/cS4AVw6KLbLG2p5JqaISDTaX+XlpteX8J+Nu3hy3AAuG5TsdKSI0AxcRFytotrLza9n8t2GXTxxVf0pb1CBi4iLVVR7uem1TL7dUMSfrxzA5Wn1p7xBBS4iLnVg5v3thiIev3IAVwyuX+UNKnARcaED5f3N+iKmXdGfK+theYMKXERcpqLay5R/LOWb9UU8dkV/rkrv7HQkx6jARcQ1DpT31+sKeezy/oyrx+UNKnARcYkD5f3V2kB5D6nf5Q0qcBFxgYpqLz8/UN5XnKLyDlCBi0hUq6j2cssbS/n32kL+dPkpXD2ki9ORooYKXESiVqXHy61vLGVhbiF/vPwUxg9VeR9MBS4iUanS4+WWfyxlQW4hf7jsFK5ReR9GBS4iUcc/885iQW4hv7+sHxNOVXnXRAUuIlGl0uPltjeymJ9TwO8u7cfEU09yOlLUUoGLSNSo9HiZOjOLeTkFPHppPyYNU3kfjQpcRKJClcfH1JlZfJldwKNj+zJZ5X1MKnARcVyVx8dtgfJ+ZGxfJp+W4nQkV1CBi4ijqjw+ps7K4svsnTwyti/XqryDpgIXEcdUeXzcPiuLL9aovI+HClxEHFHt9XHHm1l8vmYnv71E5X08VOAiEnHVXv/Me+7qnfzfxX247vQUpyO5kgpcRCKq2uvjjlnLmLt6J7+5uA/XD+/qdCTXUoGLSMQcKO/PVu/gfy/qww0q7zpRgYtIRFR7fdz5pr+8/+eiPvzsDJV3XanARSTsqr0+7nprGf/6YQe/vrA3N6q8Q0IFLiJhVe31cfdby/l0lb+8bzqzm9ORYsYxC9wY08QYs9gYs8IYs9oY89vA412NMYuMMeuMMbONMfHhjysibuIJlPcnq/L51QUq71ALZgZeCWRYawcAA4ExxphhwGPAU9baHsBu4MbwxRQRt/F4fdw121/eD1+Qys0jVN6hdswCt36lgbuNAr8skAG8E3j8NeDSsCQUEdfxeH3cPXs5n6zM56HzU5ky4mSnI8WkoN4DN8bEGWOWAwXAF8AGYI+11hPYZBvQKTwRRcRNPF4f98xZwccr8/nl+an8/CyVd7gEVeDWWq+1diCQDAwFete0WU1fa4yZYozJNMZkFhYWHn9SEYl6Hq+Pe+es4KMV23lwTCq3qLzDqlZHoVhr9wALgWFAG2NMw8BTycD2I3zNi9badGttemJiYl2yikgU83h9/OLtFXy4YjsPjOnFrSNV3uEWzFEoicaYNoHbTYFzgGxgAXBlYLPrgA/CFVJEopvXZ/nF2yv4YPl27h/di9tGdnc6Ur3Q8Nib0BF4zRgTh7/w51hrPzbGrAHeMsb8DlgGzAhjThGJUl6f5Rdzlv9U3lPPVnlHyjEL3Fq7EhhUw+Mb8b8fLiL1lNdnue/tFfxz+XbuO6+nyjvCdCamiBwXr89y/9sreH/Zj/zi3J7cntHD6Uj1jgpcRGrN67Pc/84K3lv2I/ee25M7Rqm8naACF5Fa8fosD7yzkvey/OV9p8rbMcF8iCkiAsCe8irunbOC+TkF3HOOyttpKnARCcrKbXu4bWYWO/dWaAHiKKECF5GjstYyc9EWHvloDQkt4pnz89MY1KWt07EEFbiIHEV5lYdfvf8D7y/7kbN6JvL01QNp21xXjo4WKnARqdGGwlJufWMp6wpKuffcntx+dncaNDBOx5KDqMBF5DAfr9zOg++spHGjOF7/2VDO7KHrGEUjFbiI/KTK4+MPn2bz6nd5pHVpw3MT0+jYuqnTseQIVOAiAsD2PfuZOiuLZVv28LPhXfnl+anEN9SpItFMBS4ifLW2kLveWkaVx8dzE9K4sH9HpyNJEFTgIvWY12f56/x1PDNvHT2TWvK3SWmcnNjC6VgSJBW4SD1VXFbFXW8t4+t1RVw+qBO/u6wfzeJVCW6ify2Reihry26mzsxiV2kVf7jsFK4Z2hljdIig26jAReoRay2vfZfH7z/Npn2rJrx76+mcktza6VhynFTgIvVEaaWHB99dyScr8xmVmsST4wbSulkjp2NJHajAReqBtTv3ccsbS8krKuOBMb24ZcTJOqsyBqjARWLc+8u28fB7P9C8cUNm3jSM004+welIEiIqcJEYVVHt5dGP1zBz0RaGprTj2QmDSGrVxOlYEkIqcJEYtLW4nNtmZrHqxxJ+PqIb94/uRcM4nVUZa1TgIjFmfs5O7pm9Ap+1/H3yYEb37eB0JAkTFbhIjPD6LE9+kctzCzbQp2Mrnp+UxkknNHc6loSRClwkBhTuq+Sut5bx3YZdXJ3emd+O7UuTRnFOx5IwU4GLuNySvGKmzsyiZH81067sz7j0zk5HkghRgYu4lLWW6V9v4k+f5dC5bVNevWEofU5s5XQsiSAVuIgL7a2o5v63VzB39U7G9O3AtKv606qJzqqsb1TgIi6zZvtebpu5lK279/PrC3tz4xlddSGqekoFLuIiczK38j///IHWTRvx1pRhDElp53QkcZAKXMQFKqq9/OaD1czO3MrpJ5/AM+MHkdiysdOxxGEqcJEol1dUxq0zs8jO38vtZ3fnnnN7EqcLUQkqcJGoNnf1Du6bs4IGDQwvX59ORmp7pyNJFFGBi0Shaq+Px+fm8uJXG+mf3JrnJqTRuV0zp2NJlFGBi0SZnXsruGPWMhbnFTPx1C7878V9aNxQZ1XK4VTgIlHkuw1F3PnmcsoqPTx19QAuG5TsdCSJYipwkSjg81me//cGnvg8l5SE5sy6+VR6tm/pdCyJcscscGNMZ+B1oAPgA1601j5jjGkHzAZSgDxgnLV2d/iiisSmkvJq7p2znHk5BVzYvyOPXdGfFo01t5JjC+YK7x7gF9ba3sAwYKoxpg/wS2CetbYHMC9wX0RqYdW2Ei7869d8ta6Q/7u4D89eM0jlLUE75neKtTYfyA/c3meMyQY6AWOBkYHNXgMWAg+GJaVIjLHWMmvxFn774RoSWsQz++enkdalrdOxxGVq9V+9MSYFGAQsAtoHyh1rbb4xJink6URiUHmVh1+//wPvLfuRM3sk8Mz4QbRrHu90LHGhoAvcGNMCeBe421q7N9iL5xhjpgBTALp06XI8GUVixobCUm57I4u1Bfu4+5we3JHRQ2dVynELqsCNMY3wl/dMa+17gYd3GmM6BmbfHYGCmr7WWvsi8CJAenq6DUFmEVf6ZGU+D7yzgviGDXj1hqGc1TPR6UjicsEchWKAGUC2tfbJg576ELgO+FPg9w/CklDE5ao8Pv74r2xe+TaPQV3a8NyENE5s09TpWBIDgpmBDwcmA6uMMcsDjz2Mv7jnGGNuBLYAV4Unooh75ZfsZ+rMLLK27OGG4Sk8dH5v4hsGc/CXyLEFcxTKN8CR3qQbFdo4IrHj63WF3PXWciqrvTw7YRAX9T/R6UgSY3TAqUiI+XyWv85fz9Pz1tIjqQV/mziY7kktnI4lMUgFLhJCxWVV3D17OV+tLeSyQZ34/WX9aBavl5mEh76zREJk2ZbdTJ2ZRVFpFb+/rB8ThnbRWpUSVipwkTqy1vLad3n8/tNsklo24Z1bT6N/chunY0k9oAIXqYPSSg8PvbeKj1ZsJyM1iSfHDaBNM51VKZGhAhc5Tmt37uPWN5ayqaiM+0f34tazTqaBzqqUCFKBixyHD5b/yC/fXUXzxnG8ceOpnN49welIUg+pwEVqodLj5dGP1/DG91sYktKWZyek0b5VE6djST2lAhcJ0tbicqbOymLlthKmjOjG/aN70ShOZ1WKc1TgIkFYkFPA3bOX4/NZXpg0mDH9OjgdSUQFLnI0Xp/lqS/W8uyC9fTu2IrnJ6aRktDc6VgigApc5IiKSiu5881lfLdhF1cNTubRS/vRpFGc07FEfqICF6lBZl4xU2dlsae8mmlX9GfckM5ORxI5jApc5CDWWmZ8s4k//iuH5LZNee+2IfQ9sbXTsURqpAIXCdhbUc0Db6/ks9U7OK9Pex6/agCtmzZyOpbIEanARYA12/dy28ylbN29n4cvSOXmM7vpQlQS9VTgUq9VVHt5/T95PPH5Wlo3bcSbNw9jaNd2TscSCYoKXOolj9fHu1nbePrLdeSXVDCyVyLTruxPUkudVSnuoQKXesVay2c/7ODxz3PZWFjGgM5teOKqAbqWibiSClzqjW/XFzHtsxxWbCvh5MTmvDBpMKP7ttd73eJaKnCJeSu37eHxubl8va6IE1s3YdqV/bl8UCca6jom4nIqcIlZGwpLefLztXyyKp+2zRrx6wt7M2nYSTqbUmKGClxiTn7Jfp75ch1vL91G44YNuHNUD24+systm+iYboktKnCJGXvKq3h+4QZe/S4Pn7VMHnYSt2d0J6FFY6ejiYSFClxcr7zKwyvf5vHCvzdQWunhskGduOecnnRu18zpaCJhpQIX16ry+Ji9ZAvPzFtPUWkl5/Ruz32je5LaoZXT0UQiQgUuruPzWT5auZ0nPl/LluJyhqa04++T0xh8ks6glPpFBS6uYa1lYW4hj32WQ86OffTu2IpXbhjCyJ6JOpZb6iUVuLhCZl4x0z7LZXFeMV3aNeOZ8QO5uP+JNGig4pb6SwUuUS1nx17+PDeXL7MLSGjRmEfH9uXqIV2Ib6iTcERU4BKVthaX89QXa3l/+Y+0iG/I/aN7ccPwFJrF61tW5AC9GiSqFO6r5LkF65m5aDMNjGHKmd245ayTads83uloIlFHBS5RYV9FNS99tZHp32yi0uNjXHoyd47qQcfWTZ2OJhK1VODiqIpqL298v5nnFqxnd3k1F57SkXvP68nJiS2cjiYS9VTg4giP18d7WT/y9Jdr2V5SwZk9Erh/dC/6J7dxOpqIa6jAJaKstcxdvYPH5+ayobCMAcmtefyqAQzXggoitXbMAjfGvAxcBBRYa/sFHmsHzAZSgDxgnLV2d/hiSiz4bn0Rj83NZcXWPYEFFdIY3beDTsIROU7BHEz7KjDmkMd+Ccyz1vYA5gXui9Ro1bYSJs9YxITpiyjYW8G0K/oz9+4RjOnXUeUtUgfHnIFba78yxqQc8vBYYGTg9mvAQuDBEOaSGLCxsJQnvljLJyvzaaMFFURC7njfA29vrc0HsNbmG2OSjrShMWYKMAWgS5cux7k7cZMdJRU8M28dczK3+hdUyOjOTSO60UoLKoiEVNg/xLTWvgi8CJCenm7DvT9xTk0LKkw9uzuJLbWggkg4HG+B7zTGdAzMvjsCBaEMJe5y2IIKAztxz7laUEEk3I63wD8ErgP+FPj9g5AlEteo9vp4a8lW/jJvHYX7KjmndxL3je6lBRVEIiSYwwjfxP+BZYIxZhvwG/zFPccYcyOwBbgqnCEluhxYUOHJL9ayeVc5Q1La8vzENNJTtKCCSCQFcxTKNUd4alSIs0iUs9aycG0h0z7LJTt/L6kdWvLK9UMY2UsLKog4QWdiSlCWbi7msc9yWbxJCyqIRAsVuBxV7o59PD43ly+zd5LQojGPjO3LeC2oIBIVVOBSo63F5Tz15VreX+ZfUOG+83pyw/CuNG+sbxmRaKFXo/yXotJKnp3vX1DBGMPNZ3bjVi2oIBKVVOACBBZU+HoT07/eSEW1l3HpnbnrHC2oIBLNVOD13KELKlxwSgfuPbcX3ZO0oIJItFOB11Mer4/3lv3I01/4F1Q4o7t/QYUBnbWggohbqMDrGf+CCjv58+e5rC8opX9ya6ZdOYAzemhBBRG3UYHXI99tKOKxz/wLKnRLbM7zE9MY008LKoi4lQq8Hli1rYRpc3P4el0RHVs34bErTuGKtGQaxulYbhE3U4HHsEMXVPjVBb2ZfJoWVBCJFSrwGLSxsJTp32xi9pKtxMc14I6M7tysBRVEYo4KPAZUeXwsyStmfk4B83MK2FRURqM4w6RTuzA1oztJLZs4HVFEwkAF7lKF+ypZmOsv7K/XFVFa6SG+YQNO63YC15+ewrl92nNiG52EIxLLVOAuYa1l9fa9zM8pYF5OASu37cFaaN+qMRcP6EhGanuGdz+BZvH6JxWpL/Rqj2LlVR6+WVfE/JwCFuQWsHNvJcbAgOQ23HNOTzJSk+h7YisdBihST6nAo8zW4vKfZtnfb9xFlcdHi8YNGdEzgYzU9ozslUhCCy0SLCIqcMd5vD6Wbt7N/NwC5mcXsK6gFIBuCc2ZPOwkRqUmkZ7STtffFpHDqMAdsLusin+vLWReTgH/zi1gb4WHRnGGoV3bMX5oFzJSk+ia0NzpmCIS5VTgEWCtJXfnPv9hftkFZG3Zjc9CQot4zuvbgVGpSZzRI4GWOk5bRGpBBR4mFdVe/rNhF/NydrIgp5Af9+wHoF+nVtye0YOM1CT6d2qtNSVF5LipwEMov2T/T7PsbzcUUVHto1l8HMO7J3BHRnfOTk2ifSudVCMioaECrwOvz7J86x4WBI4ayc7fC0Dndk0ZP6QLZ6cmcWrXdrr2iIiEhQq8lkr2V/P1ukLmZxewcG0hxWVVxDUwDD6pLQ+dn0pGahLdk1ro2GwRCTsV+DFYa9lQWBaYZe8kM283Hp+lTbNGnN0ribNTkzirRyKtm+kDSBGJLBV4DSo9XhZvKmZetv8MyM27ygFI7dCSKSO6Map3EgM7tyVOH0CKiINU4AEF+ypYmFPIvJydfLOuiLIqL40bNmB49wRuOrMbGalJdNLFoUQkitTbAvf5LD9sL/lplr1yWwkAHVs34dJBnchITeL0kxNoGq8PIEUkOtWrAi+tPHBxqJ0syC2kcJ//4lCDOrfh/tG9yEhNIrVDS30AKSKuEPMFvnlX2U+z7O837qLaa2nZpCFn9UwkIzWJs3omcoIuDiUiLhRzBV7t9ZGZt5v5OTuZn1PAhsIyALonteCG4V3JSE1i8EltaaQFfUXE5WKiwIvLqliY6z+Z5qu1heyr8BAf14BTu7Vj0rCTyEhN4qQTdHEoEYktrixway3Z+ft+mmUv2+pfnSaxZWMu6NeRjN5JnNE9geaNXfnXExEJimsabn+Vl+82FDEvp4AFOQXkl1QAMCC5NXeN6sGo1Pb0PbGVLg4lIvWGKwr84fdX8e7SbVR6fDSPj+PMHoncc24SI3slasV1Eam3XFHgyW2bMuHULoxKbc+Qrm1p3FDHZouI1KnAjTFjgGeAOGC6tfZPIUl1iNtGdg/HHysi4mrHfSydMSYOeA44H+gDXGOM6ROqYCIicnR1ORh6KLDeWrvRWlsFvAWMDU0sERE5lroUeCdg60H3twUe+y/GmCnGmExjTGZhYWEddiciIgerS4HXdLyePewBa1+01qZba9MTExPrsDsRETlYXQp8G9D5oPvJwPa6xRERkWDVpcCXAD2MMV2NMfHAeODD0MQSEZFjOe7DCK21HmPM7cBc/IcRvmytXR2yZCIiclR1Og7cWvsp8GmIsoiISC0Yaw/73DF8OzOmENh8nF+eABSFME6oKFftKFftKFftxGquk6y1hx0FEtECrwtjTKa1Nt3pHIdSrtpRrtpRrtqpb7m0qoGIiEupwEVEXMpNBf6i0wGOQLlqR7lqR7lqp17lcs174CIi8t/cNAMXEZGDqMBFRFwqqgrcGPOyMabAGPPDEZ43xpi/GGPWG2NWGmPSoiTXSGNMiTFmeeDX/0YoV2djzAJjTLYxZrUx5q4aton4mAWZK+JjZoxpYoxZbIxZEcj12xq2aWyMmR0Yr0XGmJQoyXW9MabwoPG6Kdy5Dtp3nDFmmTHm4xqei/h4BZnLkfEyxuQZY1YF9plZw/OhfT1aa6PmFzACSAN+OMLzFwD/wn8lxGHAoijJNRL42IHx6gikBW63BNYCfZwesyBzRXzMAmPQInC7EbAIGHbINrcBLwRujwdmR0mu64FnI/09Ftj3vcCsmv69nBivIHM5Ml5AHpBwlOdD+nqMqhm4tfYroPgom4wFXrd+3wNtjDEdoyCXI6y1+dbarMDtfUA2h1+TPeJjFmSuiAuMQWngbqPAr0M/xR8LvBa4/Q4wyhhT06WTI53LEcaYZOBCYPoRNon4eAWZK1qF9PUYVQUehKAWkXDIaYEfgf9ljOkb6Z0HfnQdhH/2djBHx+woucCBMQv82L0cKAC+sNYecbystR6gBDghCnIBXBH4sfsdY0znGp4Ph6eBBwDfEZ53ZLyCyAXOjJcFPjfGLDXGTKnh+ZC+Ht1W4EEtIuGALPzXKhgA/BX4ZyR3boxpAbwL3G2t3Xvo0zV8SUTG7Bi5HBkza63XWjsQ//Xrhxpj+h2yiSPjFUSuj4AUa21/4Ev+/6w3bIwxFwEF1tqlR9ushsfCOl5B5or4eAUMt9am4V8reKoxZsQhz4d0vNxW4FG5iIS1du+BH4Gt/wqNjYwxCZHYtzGmEf6SnGmtfa+GTRwZs2PlcnLMAvvcAywExhzy1E/jZYxpCLQmgm+fHSmXtXaXtbYycPclYHAE4gwHLjHG5OFf8zbDGPPGIds4MV7HzOXQeGGt3R74vQB4H//awQcL6evRbQX+IXBt4JPcYUCJtTbf6VDGmA4H3vczxgzFP667IrBfA8wAsq21Tx5hs4iPWTC5nBgzY0yiMaZN4HZT4Bwg55DNPgSuC9y+EphvA58+OZnrkPdJL8H/uUJYWWsfstYmW2tT8H9AOd9aO+mQzSI+XsHkcmK8jDHNjTEtD9wGzgMOPXItpK/HOl0PPNSMMW/iPzohwRizDfgN/g90sNa+gP/a4xcA64Fy4IYoyXUlcKsxxgPsB8aH+5s4YDgwGVgVeP8U4GGgy0HZnBizYHI5MWYdgdeMMXH4/8OYY6392BjzCJBprf0Q/388/zDGrMc/kxwf5kzB5rrTGHMJ4Ankuj4CuWoUBeMVTC4nxqs98H5gXtIQmGWt/cwYcwuE5/WoU+lFRFzKbW+hiIhIgApcRMSlVOAiIi6lAhcRcSkVuIiIS6nARURcSgUuIuJS/w+n/yNQ3mxnpQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x_data,y_data)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize w2 and w1 with randon values\n",
    "\n",
    "w_1 = torch.tensor([1.0], requires_grad=True)\n",
    "w_2 = torch.tensor([1.0], requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Quadratic forward pass based on the function above. Taking b as zero for now\n",
    "\n",
    "def quad_forward(x):\n",
    "    return w_1*(x**2)+w_2*x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loss fucntion as per the defination above\n",
    "\n",
    "def loss(y_pred,y):\n",
    "    return (y_pred-y)**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict (before training) 6 42.0\n",
      "\tgrad:  1.0 1.0 2.0 2.0\n",
      "\tgrad:  2.0 6.0 -0.11520004272460938 -0.11520004272460938\n",
      "\tgrad:  3.0 15.0 -54.492271423339844 -54.492271423339844\n",
      "\tgrad:  4.0 28 -221.18634033203125 -221.18634033203125\n",
      "\tgrad:  5.0 45 -317.8537292480469 -317.8537292480469\n",
      "Epoch: 0 | Loss: 40.41239929199219 | w1: 1.7099769115447998 | w2: 1.1621068716049194\n",
      "\tgrad:  1.0 1.0 3.7441673278808594 3.7441673278808594\n",
      "\tgrad:  2.0 6.0 25.097305297851562 25.097305297851562\n",
      "\tgrad:  3.0 15.0 63.107460021972656 63.107460021972656\n",
      "\tgrad:  4.0 28 66.03076171875 66.03076171875\n",
      "\tgrad:  5.0 45 -75.12264251708984 -75.12264251708984\n",
      "Epoch: 1 | Loss: 2.257364511489868 | w1: 1.610548496246338 | w2: 1.115532636642456\n",
      "\tgrad:  1.0 1.0 3.452162265777588 3.452162265777588\n",
      "\tgrad:  2.0 6.0 21.187225341796875 21.187225341796875\n",
      "\tgrad:  3.0 15.0 45.44755554199219 45.44755554199219\n",
      "\tgrad:  4.0 28 23.84320068359375 23.84320068359375\n",
      "\tgrad:  5.0 45 -109.37328338623047 -109.37328338623047\n",
      "Epoch: 2 | Loss: 4.785006046295166 | w1: 1.629080057144165 | w2: 1.099595308303833\n",
      "\tgrad:  1.0 1.0 3.457350730895996 3.457350730895996\n",
      "\tgrad:  2.0 6.0 21.524940490722656 21.524940490722656\n",
      "\tgrad:  3.0 15.0 47.5110969543457 47.5110969543457\n",
      "\tgrad:  4.0 28 29.6805419921875 29.6805419921875\n",
      "\tgrad:  5.0 45 -103.25508117675781 -103.25508117675781\n",
      "Epoch: 3 | Loss: 4.264644622802734 | w1: 1.6303775310516357 | w2: 1.0794039964675903\n",
      "\tgrad:  1.0 1.0 3.4195632934570312 3.4195632934570312\n",
      "\tgrad:  2.0 6.0 21.245582580566406 21.245582580566406\n",
      "\tgrad:  3.0 15.0 46.70415115356445 46.70415115356445\n",
      "\tgrad:  4.0 28 28.51959228515625 28.51959228515625\n",
      "\tgrad:  5.0 45 -103.0324935913086 -103.0324935913086\n",
      "Epoch: 4 | Loss: 4.246277809143066 | w1: 1.6341499090194702 | w2: 1.060043454170227\n",
      "\tgrad:  1.0 1.0 3.3883867263793945 3.3883867263793945\n",
      "\tgrad:  2.0 6.0 21.058319091796875 21.058319091796875\n",
      "\tgrad:  3.0 15.0 46.320316314697266 46.320316314697266\n",
      "\tgrad:  4.0 28 28.38165283203125 28.38165283203125\n",
      "\tgrad:  5.0 45 -101.96094512939453 -101.96094512939453\n",
      "Epoch: 5 | Loss: 4.158413887023926 | w1: 1.6375246047973633 | w2: 1.0407702922821045\n",
      "\tgrad:  1.0 1.0 3.3565897941589355 3.3565897941589355\n",
      "\tgrad:  2.0 6.0 20.859771728515625 20.859771728515625\n",
      "\tgrad:  3.0 15.0 45.87955856323242 45.87955856323242\n",
      "\tgrad:  4.0 28 28.09759521484375 28.09759521484375\n",
      "\tgrad:  5.0 45 -101.02348327636719 -101.02348327636719\n",
      "Epoch: 6 | Loss: 4.0822978019714355 | w1: 1.6409205198287964 | w2: 1.0216909646987915\n",
      "\tgrad:  1.0 1.0 3.325222969055176 3.325222969055176\n",
      "\tgrad:  2.0 6.0 20.66497802734375 20.66497802734375\n",
      "\tgrad:  3.0 15.0 45.45170974731445 45.45170974731445\n",
      "\tgrad:  4.0 28 27.83770751953125 27.83770751953125\n",
      "\tgrad:  5.0 45 -100.07648468017578 -100.07648468017578\n",
      "Epoch: 7 | Loss: 4.0061211585998535 | w1: 1.644276738166809 | w2: 1.0027880668640137\n",
      "\tgrad:  1.0 1.0 3.2941293716430664 3.2941293716430664\n",
      "\tgrad:  2.0 6.0 20.471725463867188 20.471725463867188\n",
      "\tgrad:  3.0 15.0 45.026573181152344 45.026573181152344\n",
      "\tgrad:  4.0 28 27.57696533203125 27.57696533203125\n",
      "\tgrad:  5.0 45 -99.14112091064453 -99.14112091064453\n",
      "Epoch: 8 | Loss: 3.9315848350524902 | w1: 1.6476027965545654 | w2: 0.9840622544288635\n",
      "\tgrad:  1.0 1.0 3.2633299827575684 3.2633299827575684\n",
      "\tgrad:  2.0 6.0 20.280319213867188 20.280319213867188\n",
      "\tgrad:  3.0 15.0 44.6055908203125 44.6055908203125\n",
      "\tgrad:  4.0 28 27.319091796875 27.319091796875\n",
      "\tgrad:  5.0 45 -98.21395874023438 -98.21395874023438\n",
      "Epoch: 9 | Loss: 3.8583927154541016 | w1: 1.65089750289917 | w2: 0.9655115008354187\n",
      "\tgrad:  1.0 1.0 3.232818126678467 3.232818126678467\n",
      "\tgrad:  2.0 6.0 20.09069061279297 20.09069061279297\n",
      "\tgrad:  3.0 15.0 44.1884880065918 44.1884880065918\n",
      "\tgrad:  4.0 28 27.0635986328125 27.0635986328125\n",
      "\tgrad:  5.0 45 -97.29595184326172 -97.29595184326172\n",
      "Epoch: 10 | Loss: 3.7866008281707764 | w1: 1.654161810874939 | w2: 0.9471341967582703\n",
      "\tgrad:  1.0 1.0 3.202591896057129 3.202591896057129\n",
      "\tgrad:  2.0 6.0 19.902854919433594 19.902854919433594\n",
      "\tgrad:  3.0 15.0 43.775367736816406 43.775367736816406\n",
      "\tgrad:  4.0 28 26.81072998046875 26.81072998046875\n",
      "\tgrad:  5.0 45 -96.38595581054688 -96.38595581054688\n",
      "Epoch: 11 | Loss: 3.7161009311676025 | w1: 1.6573951244354248 | w2: 0.928928554058075\n",
      "\tgrad:  1.0 1.0 3.172647476196289 3.172647476196289\n",
      "\tgrad:  2.0 6.0 19.716758728027344 19.716758728027344\n",
      "\tgrad:  3.0 15.0 43.366058349609375 43.366058349609375\n",
      "\tgrad:  4.0 28 26.5599365234375 26.5599365234375\n",
      "\tgrad:  5.0 45 -95.48492431640625 -95.48492431640625\n",
      "Epoch: 12 | Loss: 3.6469483375549316 | w1: 1.660598635673523 | w2: 0.9108932614326477\n",
      "\tgrad:  1.0 1.0 3.142983913421631 3.142983913421631\n",
      "\tgrad:  2.0 6.0 19.53240966796875 19.53240966796875\n",
      "\tgrad:  3.0 15.0 42.960594177246094 42.960594177246094\n",
      "\tgrad:  4.0 28 26.3116455078125 26.3116455078125\n",
      "\tgrad:  5.0 45 -94.59190368652344 -94.59190368652344\n",
      "Epoch: 13 | Loss: 3.5790512561798096 | w1: 1.6637717485427856 | w2: 0.8930264711380005\n",
      "\tgrad:  1.0 1.0 3.1135964393615723 3.1135964393615723\n",
      "\tgrad:  2.0 6.0 19.349777221679688 19.349777221679688\n",
      "\tgrad:  3.0 15.0 42.55890655517578 42.55890655517578\n",
      "\tgrad:  4.0 28 26.0655517578125 26.0655517578125\n",
      "\tgrad:  5.0 45 -93.70746612548828 -93.70746612548828\n",
      "Epoch: 14 | Loss: 3.5124356746673584 | w1: 1.6669154167175293 | w2: 0.8753268718719482\n",
      "\tgrad:  1.0 1.0 3.084484577178955 3.084484577178955\n",
      "\tgrad:  2.0 6.0 19.168861389160156 19.168861389160156\n",
      "\tgrad:  3.0 15.0 42.16096115112305 42.16096115112305\n",
      "\tgrad:  4.0 28 25.82183837890625 25.82183837890625\n",
      "\tgrad:  5.0 45 -92.8314208984375 -92.8314208984375\n",
      "Epoch: 15 | Loss: 3.4470691680908203 | w1: 1.670029878616333 | w2: 0.8577927947044373\n",
      "\tgrad:  1.0 1.0 3.05564546585083 3.05564546585083\n",
      "\tgrad:  2.0 6.0 18.989639282226562 18.989639282226562\n",
      "\tgrad:  3.0 15.0 41.76679229736328 41.76679229736328\n",
      "\tgrad:  4.0 28 25.5804443359375 25.5804443359375\n",
      "\tgrad:  5.0 45 -91.96357727050781 -91.96357727050781\n",
      "Epoch: 16 | Loss: 3.3829197883605957 | w1: 1.6731152534484863 | w2: 0.8404226303100586\n",
      "\tgrad:  1.0 1.0 3.02707576751709 3.02707576751709\n",
      "\tgrad:  2.0 6.0 18.812088012695312 18.812088012695312\n",
      "\tgrad:  3.0 15.0 41.37626266479492 41.37626266479492\n",
      "\tgrad:  4.0 28 25.34130859375 25.34130859375\n",
      "\tgrad:  5.0 45 -91.10374450683594 -91.10374450683594\n",
      "Epoch: 17 | Loss: 3.3199570178985596 | w1: 1.6761715412139893 | w2: 0.8232148885726929\n",
      "\tgrad:  1.0 1.0 2.998772621154785 2.998772621154785\n",
      "\tgrad:  2.0 6.0 18.636199951171875 18.636199951171875\n",
      "\tgrad:  3.0 15.0 40.98940658569336 40.98940658569336\n",
      "\tgrad:  4.0 28 25.1043701171875 25.1043701171875\n",
      "\tgrad:  5.0 45 -90.25192260742188 -90.25192260742188\n",
      "Epoch: 18 | Loss: 3.2581639289855957 | w1: 1.6791993379592896 | w2: 0.8061680793762207\n",
      "\tgrad:  1.0 1.0 2.9707345962524414 2.9707345962524414\n",
      "\tgrad:  2.0 6.0 18.461952209472656 18.461952209472656\n",
      "\tgrad:  3.0 15.0 40.6061897277832 40.6061897277832\n",
      "\tgrad:  4.0 28 24.86968994140625 24.86968994140625\n",
      "\tgrad:  5.0 45 -89.40773010253906 -89.40773010253906\n",
      "Epoch: 19 | Loss: 3.1974968910217285 | w1: 1.6821985244750977 | w2: 0.7892804741859436\n",
      "\tgrad:  1.0 1.0 2.942957878112793 2.942957878112793\n",
      "\tgrad:  2.0 6.0 18.289321899414062 18.289321899414062\n",
      "\tgrad:  3.0 15.0 40.22647476196289 40.22647476196289\n",
      "\tgrad:  4.0 28 24.63702392578125 24.63702392578125\n",
      "\tgrad:  5.0 45 -88.57192993164062 -88.57192993164062\n",
      "Epoch: 20 | Loss: 3.1379947662353516 | w1: 1.6851699352264404 | w2: 0.7725509405136108\n",
      "\tgrad:  1.0 1.0 2.9154415130615234 2.9154415130615234\n",
      "\tgrad:  2.0 6.0 18.118324279785156 18.118324279785156\n",
      "\tgrad:  3.0 15.0 39.850364685058594 39.850364685058594\n",
      "\tgrad:  4.0 28 24.40667724609375 24.40667724609375\n",
      "\tgrad:  5.0 45 -87.744140625 -87.744140625\n",
      "Epoch: 21 | Loss: 3.07961368560791 | w1: 1.688113808631897 | w2: 0.7559778690338135\n",
      "\tgrad:  1.0 1.0 2.88818359375 2.88818359375\n",
      "\tgrad:  2.0 6.0 17.948928833007812 17.948928833007812\n",
      "\tgrad:  3.0 15.0 39.47779083251953 39.47779083251953\n",
      "\tgrad:  4.0 28 24.17864990234375 24.17864990234375\n",
      "\tgrad:  5.0 45 -86.92340850830078 -86.92340850830078\n",
      "Epoch: 22 | Loss: 3.0222716331481934 | w1: 1.6910297870635986 | w2: 0.7395595908164978\n",
      "\tgrad:  1.0 1.0 2.8611788749694824 2.8611788749694824\n",
      "\tgrad:  2.0 6.0 17.781097412109375 17.781097412109375\n",
      "\tgrad:  3.0 15.0 39.10865020751953 39.10865020751953\n",
      "\tgrad:  4.0 28 23.95245361328125 23.95245361328125\n",
      "\tgrad:  5.0 45 -86.11087799072266 -86.11087799072266\n",
      "Epoch: 23 | Loss: 2.9660332202911377 | w1: 1.6939187049865723 | w2: 0.7232949137687683\n",
      "\tgrad:  1.0 1.0 2.8344273567199707 2.8344273567199707\n",
      "\tgrad:  2.0 6.0 17.614852905273438 17.614852905273438\n",
      "\tgrad:  3.0 15.0 38.743011474609375 38.743011474609375\n",
      "\tgrad:  4.0 28 23.72857666015625 23.72857666015625\n",
      "\tgrad:  5.0 45 -85.3057861328125 -85.3057861328125\n",
      "Epoch: 24 | Loss: 2.9108309745788574 | w1: 1.6967805624008179 | w2: 0.7071822285652161\n",
      "\tgrad:  1.0 1.0 2.8079257011413574 2.8079257011413574\n",
      "\tgrad:  2.0 6.0 17.450157165527344 17.450157165527344\n",
      "\tgrad:  3.0 15.0 38.38077163696289 38.38077163696289\n",
      "\tgrad:  4.0 28 23.50665283203125 23.50665283203125\n",
      "\tgrad:  5.0 45 -84.50813293457031 -84.50813293457031\n",
      "Epoch: 25 | Loss: 2.856649875640869 | w1: 1.6996155977249146 | w2: 0.6912202835083008\n",
      "\tgrad:  1.0 1.0 2.7816715240478516 2.7816715240478516\n",
      "\tgrad:  2.0 6.0 17.287002563476562 17.287002563476562\n",
      "\tgrad:  3.0 15.0 38.02193069458008 38.02193069458008\n",
      "\tgrad:  4.0 28 23.286865234375 23.286865234375\n",
      "\tgrad:  5.0 45 -83.71772766113281 -83.71772766113281\n",
      "Epoch: 26 | Loss: 2.8034632205963135 | w1: 1.7024239301681519 | w2: 0.6754074692726135\n",
      "\tgrad:  1.0 1.0 2.7556629180908203 2.7556629180908203\n",
      "\tgrad:  2.0 6.0 17.12535858154297 17.12535858154297\n",
      "\tgrad:  3.0 15.0 37.666385650634766 37.666385650634766\n",
      "\tgrad:  4.0 28 23.069091796875 23.069091796875\n",
      "\tgrad:  5.0 45 -82.93514251708984 -82.93514251708984\n",
      "Epoch: 27 | Loss: 2.7512950897216797 | w1: 1.7052063941955566 | w2: 0.659742534160614\n",
      "\tgrad:  1.0 1.0 2.729897975921631 2.729897975921631\n",
      "\tgrad:  2.0 6.0 16.965240478515625 16.965240478515625\n",
      "\tgrad:  3.0 15.0 37.314205169677734 37.314205169677734\n",
      "\tgrad:  4.0 28 22.85345458984375 22.85345458984375\n",
      "\tgrad:  5.0 45 -82.15980529785156 -82.15980529785156\n",
      "Epoch: 28 | Loss: 2.7000935077667236 | w1: 1.7079628705978394 | w2: 0.6442241668701172\n",
      "\tgrad:  1.0 1.0 2.704374313354492 2.704374313354492\n",
      "\tgrad:  2.0 6.0 16.806625366210938 16.806625366210938\n",
      "\tgrad:  3.0 15.0 36.965354919433594 36.965354919433594\n",
      "\tgrad:  4.0 28 22.639892578125 22.639892578125\n",
      "\tgrad:  5.0 45 -81.39152526855469 -81.39152526855469\n",
      "Epoch: 29 | Loss: 2.649832248687744 | w1: 1.7106932401657104 | w2: 0.6288508772850037\n",
      "\tgrad:  1.0 1.0 2.6790881156921387 2.6790881156921387\n",
      "\tgrad:  2.0 6.0 16.64948272705078 16.64948272705078\n",
      "\tgrad:  3.0 15.0 36.61973190307617 36.61973190307617\n",
      "\tgrad:  4.0 28 22.4281005859375 22.4281005859375\n",
      "\tgrad:  5.0 45 -80.6304931640625 -80.6304931640625\n",
      "Epoch: 30 | Loss: 2.600510597229004 | w1: 1.7133980989456177 | w2: 0.6136212944984436\n",
      "\tgrad:  1.0 1.0 2.654038906097412 2.654038906097412\n",
      "\tgrad:  2.0 6.0 16.493804931640625 16.493804931640625\n",
      "\tgrad:  3.0 15.0 36.27733612060547 36.27733612060547\n",
      "\tgrad:  4.0 28 22.21832275390625 22.21832275390625\n",
      "\tgrad:  5.0 45 -79.876708984375 -79.876708984375\n",
      "Epoch: 31 | Loss: 2.5521154403686523 | w1: 1.7160779237747192 | w2: 0.5985341668128967\n",
      "\tgrad:  1.0 1.0 2.6292243003845215 2.6292243003845215\n",
      "\tgrad:  2.0 6.0 16.339599609375 16.339599609375\n",
      "\tgrad:  3.0 15.0 35.938133239746094 35.938133239746094\n",
      "\tgrad:  4.0 28 22.0106201171875 22.0106201171875\n",
      "\tgrad:  5.0 45 -79.12979125976562 -79.12979125976562\n",
      "Epoch: 32 | Loss: 2.5046095848083496 | w1: 1.7187325954437256 | w2: 0.5835880637168884\n",
      "\tgrad:  1.0 1.0 2.6046414375305176 2.6046414375305176\n",
      "\tgrad:  2.0 6.0 16.18682861328125 16.18682861328125\n",
      "\tgrad:  3.0 15.0 35.60215759277344 35.60215759277344\n",
      "\tgrad:  4.0 28 21.80487060546875 21.80487060546875\n",
      "\tgrad:  5.0 45 -78.38993072509766 -78.38993072509766\n",
      "Epoch: 33 | Loss: 2.4579925537109375 | w1: 1.7213623523712158 | w2: 0.5687816739082336\n",
      "\tgrad:  1.0 1.0 2.5802879333496094 2.5802879333496094\n",
      "\tgrad:  2.0 6.0 16.035476684570312 16.035476684570312\n",
      "\tgrad:  3.0 15.0 35.26927185058594 35.26927185058594\n",
      "\tgrad:  4.0 28 21.60101318359375 21.60101318359375\n",
      "\tgrad:  5.0 45 -77.6571273803711 -77.6571273803711\n",
      "Epoch: 34 | Loss: 2.4122517108917236 | w1: 1.7239676713943481 | w2: 0.5541138052940369\n",
      "\tgrad:  1.0 1.0 2.5561628341674805 2.5561628341674805\n",
      "\tgrad:  2.0 6.0 15.885551452636719 15.885551452636719\n",
      "\tgrad:  3.0 15.0 34.939510345458984 34.939510345458984\n",
      "\tgrad:  4.0 28 21.3990478515625 21.3990478515625\n",
      "\tgrad:  5.0 45 -76.93099975585938 -76.93099975585938\n",
      "Epoch: 35 | Loss: 2.367351531982422 | w1: 1.7265485525131226 | w2: 0.5395830273628235\n",
      "\tgrad:  1.0 1.0 2.5322632789611816 2.5322632789611816\n",
      "\tgrad:  2.0 6.0 15.737022399902344 15.737022399902344\n",
      "\tgrad:  3.0 15.0 34.6128044128418 34.6128044128418\n",
      "\tgrad:  4.0 28 21.198974609375 21.198974609375\n",
      "\tgrad:  5.0 45 -76.21173858642578 -76.21173858642578\n",
      "Epoch: 36 | Loss: 2.323291540145874 | w1: 1.7291052341461182 | w2: 0.5251880884170532\n",
      "\tgrad:  1.0 1.0 2.508586883544922 2.508586883544922\n",
      "\tgrad:  2.0 6.0 15.589881896972656 15.589881896972656\n",
      "\tgrad:  3.0 15.0 34.289188385009766 34.289188385009766\n",
      "\tgrad:  4.0 28 21.00079345703125 21.00079345703125\n",
      "\tgrad:  5.0 45 -75.49915313720703 -75.49915313720703\n",
      "Epoch: 37 | Loss: 2.2800488471984863 | w1: 1.7316380739212036 | w2: 0.510927677154541\n",
      "\tgrad:  1.0 1.0 2.48513126373291 2.48513126373291\n",
      "\tgrad:  2.0 6.0 15.44411849975586 15.44411849975586\n",
      "\tgrad:  3.0 15.0 33.96859359741211 33.96859359741211\n",
      "\tgrad:  4.0 28 20.80438232421875 20.80438232421875\n",
      "\tgrad:  5.0 45 -74.79324340820312 -74.79324340820312\n",
      "Epoch: 38 | Loss: 2.237611770629883 | w1: 1.7341471910476685 | w2: 0.4968006908893585\n",
      "\tgrad:  1.0 1.0 2.4618959426879883 2.4618959426879883\n",
      "\tgrad:  2.0 6.0 15.29971694946289 15.29971694946289\n",
      "\tgrad:  3.0 15.0 33.65098571777344 33.65098571777344\n",
      "\tgrad:  4.0 28 20.60986328125 20.60986328125\n",
      "\tgrad:  5.0 45 -74.09400939941406 -74.09400939941406\n",
      "Epoch: 39 | Loss: 2.1959688663482666 | w1: 1.736633062362671 | w2: 0.4828057885169983\n",
      "\tgrad:  1.0 1.0 2.438877582550049 2.438877582550049\n",
      "\tgrad:  2.0 6.0 15.156669616699219 15.156669616699219\n",
      "\tgrad:  3.0 15.0 33.33636474609375 33.33636474609375\n",
      "\tgrad:  4.0 28 20.41717529296875 20.41717529296875\n",
      "\tgrad:  5.0 45 -73.40106964111328 -73.40106964111328\n",
      "Epoch: 40 | Loss: 2.1550867557525635 | w1: 1.7390953302383423 | w2: 0.46894168853759766\n",
      "\tgrad:  1.0 1.0 2.416073799133301 2.416073799133301\n",
      "\tgrad:  2.0 6.0 15.01495361328125 15.01495361328125\n",
      "\tgrad:  3.0 15.0 33.024662017822266 33.024662017822266\n",
      "\tgrad:  4.0 28 20.226318359375 20.226318359375\n",
      "\tgrad:  5.0 45 -72.71480560302734 -72.71480560302734\n",
      "Epoch: 41 | Loss: 2.1149771213531494 | w1: 1.74153470993042 | w2: 0.4552072286605835\n",
      "\tgrad:  1.0 1.0 2.393484115600586 2.393484115600586\n",
      "\tgrad:  2.0 6.0 14.874561309814453 14.874561309814453\n",
      "\tgrad:  3.0 15.0 32.715877532958984 32.715877532958984\n",
      "\tgrad:  4.0 28 20.037109375 20.037109375\n",
      "\tgrad:  5.0 45 -72.03502655029297 -72.03502655029297\n",
      "Epoch: 42 | Loss: 2.075618028640747 | w1: 1.743951439857483 | w2: 0.4416012763977051\n",
      "\tgrad:  1.0 1.0 2.371105194091797 2.371105194091797\n",
      "\tgrad:  2.0 6.0 14.735492706298828 14.735492706298828\n",
      "\tgrad:  3.0 15.0 32.410011291503906 32.410011291503906\n",
      "\tgrad:  4.0 28 19.849853515625 19.849853515625\n",
      "\tgrad:  5.0 45 -71.3613510131836 -71.3613510131836\n",
      "Epoch: 43 | Loss: 2.0369770526885986 | w1: 1.7463454008102417 | w2: 0.4281224012374878\n",
      "\tgrad:  1.0 1.0 2.348935604095459 2.348935604095459\n",
      "\tgrad:  2.0 6.0 14.597713470458984 14.597713470458984\n",
      "\tgrad:  3.0 15.0 32.10696029663086 32.10696029663086\n",
      "\tgrad:  4.0 28 19.66424560546875 19.66424560546875\n",
      "\tgrad:  5.0 45 -70.69435119628906 -70.69435119628906\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 | Loss: 1.99907648563385 | w1: 1.7487173080444336 | w2: 0.41476961970329285\n",
      "\tgrad:  1.0 1.0 2.3269739151000977 2.3269739151000977\n",
      "\tgrad:  2.0 6.0 14.461235046386719 14.461235046386719\n",
      "\tgrad:  3.0 15.0 31.806793212890625 31.806793212890625\n",
      "\tgrad:  4.0 28 19.48040771484375 19.48040771484375\n",
      "\tgrad:  5.0 45 -70.03307342529297 -70.03307342529297\n",
      "Epoch: 45 | Loss: 1.9618525505065918 | w1: 1.7510664463043213 | w2: 0.4015416204929352\n",
      "\tgrad:  1.0 1.0 2.3052163124084473 2.3052163124084473\n",
      "\tgrad:  2.0 6.0 14.326011657714844 14.326011657714844\n",
      "\tgrad:  3.0 15.0 31.50937271118164 31.50937271118164\n",
      "\tgrad:  4.0 28 19.29815673828125 19.29815673828125\n",
      "\tgrad:  5.0 45 -69.37847137451172 -69.37847137451172\n",
      "Epoch: 46 | Loss: 1.9253488779067993 | w1: 1.7533941268920898 | w2: 0.3884373605251312\n",
      "\tgrad:  1.0 1.0 2.283662796020508 2.283662796020508\n",
      "\tgrad:  2.0 6.0 14.192070007324219 14.192070007324219\n",
      "\tgrad:  3.0 15.0 31.214801788330078 31.214801788330078\n",
      "\tgrad:  4.0 28 19.11785888671875 19.11785888671875\n",
      "\tgrad:  5.0 45 -68.72978210449219 -68.72978210449219\n",
      "Epoch: 47 | Loss: 1.8895131349563599 | w1: 1.755699872970581 | w2: 0.3754555881023407\n",
      "\tgrad:  1.0 1.0 2.2623109817504883 2.2623109817504883\n",
      "\tgrad:  2.0 6.0 14.059375762939453 14.059375762939453\n",
      "\tgrad:  3.0 15.0 30.922943115234375 30.922943115234375\n",
      "\tgrad:  4.0 28 18.93902587890625 18.93902587890625\n",
      "\tgrad:  5.0 45 -68.08700561523438 -68.08700561523438\n",
      "Epoch: 48 | Loss: 1.854336142539978 | w1: 1.757983922958374 | w2: 0.36259517073631287\n",
      "\tgrad:  1.0 1.0 2.2411580085754395 2.2411580085754395\n",
      "\tgrad:  2.0 6.0 13.92791748046875 13.92791748046875\n",
      "\tgrad:  3.0 15.0 30.63376235961914 30.63376235961914\n",
      "\tgrad:  4.0 28 18.76190185546875 18.76190185546875\n",
      "\tgrad:  5.0 45 -67.45033264160156 -67.45033264160156\n",
      "Epoch: 49 | Loss: 1.8198189735412598 | w1: 1.7602466344833374 | w2: 0.34985506534576416\n",
      "\tgrad:  1.0 1.0 2.220203399658203 2.220203399658203\n",
      "\tgrad:  2.0 6.0 13.797691345214844 13.797691345214844\n",
      "\tgrad:  3.0 15.0 30.347328186035156 30.347328186035156\n",
      "\tgrad:  4.0 28 18.58648681640625 18.58648681640625\n",
      "\tgrad:  5.0 45 -66.81976318359375 -66.81976318359375\n",
      "Epoch: 50 | Loss: 1.7859523296356201 | w1: 1.7624883651733398 | w2: 0.3372340798377991\n",
      "\tgrad:  1.0 1.0 2.1994447708129883 2.1994447708129883\n",
      "\tgrad:  2.0 6.0 13.668685913085938 13.668685913085938\n",
      "\tgrad:  3.0 15.0 30.06360626220703 30.06360626220703\n",
      "\tgrad:  4.0 28 18.4127197265625 18.4127197265625\n",
      "\tgrad:  5.0 45 -66.19510650634766 -66.19510650634766\n",
      "Epoch: 51 | Loss: 1.7527168989181519 | w1: 1.764709234237671 | w2: 0.32473108172416687\n",
      "\tgrad:  1.0 1.0 2.1788806915283203 2.1788806915283203\n",
      "\tgrad:  2.0 6.0 13.540889739990234 13.540889739990234\n",
      "\tgrad:  3.0 15.0 29.782562255859375 29.782562255859375\n",
      "\tgrad:  4.0 28 18.2406005859375 18.2406005859375\n",
      "\tgrad:  5.0 45 -65.576171875 -65.576171875\n",
      "Epoch: 52 | Loss: 1.7200937271118164 | w1: 1.766909122467041 | w2: 0.3123449683189392\n",
      "\tgrad:  1.0 1.0 2.15850830078125 2.15850830078125\n",
      "\tgrad:  2.0 6.0 13.414283752441406 13.414283752441406\n",
      "\tgrad:  3.0 15.0 29.504058837890625 29.504058837890625\n",
      "\tgrad:  4.0 28 18.07000732421875 18.07000732421875\n",
      "\tgrad:  5.0 45 -64.96315002441406 -64.96315002441406\n",
      "Epoch: 53 | Loss: 1.688084363937378 | w1: 1.7690885066986084 | w2: 0.3000746965408325\n",
      "\tgrad:  1.0 1.0 2.138326644897461 2.138326644897461\n",
      "\tgrad:  2.0 6.0 13.288860321044922 13.288860321044922\n",
      "\tgrad:  3.0 15.0 29.228199005126953 29.228199005126953\n",
      "\tgrad:  4.0 28 17.90106201171875 17.90106201171875\n",
      "\tgrad:  5.0 45 -64.35565948486328 -64.35565948486328\n",
      "Epoch: 54 | Loss: 1.6566603183746338 | w1: 1.7712475061416626 | w2: 0.28791916370391846\n",
      "\tgrad:  1.0 1.0 2.118333339691162 2.118333339691162\n",
      "\tgrad:  2.0 6.0 13.16461181640625 13.16461181640625\n",
      "\tgrad:  3.0 15.0 28.954914093017578 28.954914093017578\n",
      "\tgrad:  4.0 28 17.73370361328125 17.73370361328125\n",
      "\tgrad:  5.0 45 -63.753700256347656 -63.753700256347656\n",
      "Epoch: 55 | Loss: 1.6258137226104736 | w1: 1.7733861207962036 | w2: 0.2758772373199463\n",
      "\tgrad:  1.0 1.0 2.098526954650879 2.098526954650879\n",
      "\tgrad:  2.0 6.0 13.041519165039062 13.041519165039062\n",
      "\tgrad:  3.0 15.0 28.6842041015625 28.6842041015625\n",
      "\tgrad:  4.0 28 17.56781005859375 17.56781005859375\n",
      "\tgrad:  5.0 45 -63.15784454345703 -63.15784454345703\n",
      "Epoch: 56 | Loss: 1.5955653190612793 | w1: 1.7755051851272583 | w2: 0.2639479637145996\n",
      "\tgrad:  1.0 1.0 2.0789060592651367 2.0789060592651367\n",
      "\tgrad:  2.0 6.0 12.919586181640625 12.919586181640625\n",
      "\tgrad:  3.0 15.0 28.416000366210938 28.416000366210938\n",
      "\tgrad:  4.0 28 17.4036865234375 17.4036865234375\n",
      "\tgrad:  5.0 45 -62.56732940673828 -62.56732940673828\n",
      "Epoch: 57 | Loss: 1.5658682584762573 | w1: 1.777604103088379 | w2: 0.2521301805973053\n",
      "\tgrad:  1.0 1.0 2.0594687461853027 2.0594687461853027\n",
      "\tgrad:  2.0 6.0 12.798789978027344 12.798789978027344\n",
      "\tgrad:  3.0 15.0 28.15030288696289 28.15030288696289\n",
      "\tgrad:  4.0 28 17.240966796875 17.240966796875\n",
      "\tgrad:  5.0 45 -61.98234558105469 -61.98234558105469\n",
      "Epoch: 58 | Loss: 1.5367244482040405 | w1: 1.7796835899353027 | w2: 0.24042290449142456\n",
      "\tgrad:  1.0 1.0 2.040213108062744 2.040213108062744\n",
      "\tgrad:  2.0 6.0 12.679126739501953 12.679126739501953\n",
      "\tgrad:  3.0 15.0 27.88714599609375 27.88714599609375\n",
      "\tgrad:  4.0 28 17.079833984375 17.079833984375\n",
      "\tgrad:  5.0 45 -61.40270233154297 -61.40270233154297\n",
      "Epoch: 59 | Loss: 1.5081167221069336 | w1: 1.7817434072494507 | w2: 0.22882501780986786\n",
      "\tgrad:  1.0 1.0 2.02113676071167 2.02113676071167\n",
      "\tgrad:  2.0 6.0 12.56057357788086 12.56057357788086\n",
      "\tgrad:  3.0 15.0 27.626358032226562 27.626358032226562\n",
      "\tgrad:  4.0 28 16.9200439453125 16.9200439453125\n",
      "\tgrad:  5.0 45 -60.82878112792969 -60.82878112792969\n",
      "Epoch: 60 | Loss: 1.4800562858581543 | w1: 1.78378427028656 | w2: 0.21733567118644714\n",
      "\tgrad:  1.0 1.0 2.00223970413208 2.00223970413208\n",
      "\tgrad:  2.0 6.0 12.443138122558594 12.443138122558594\n",
      "\tgrad:  3.0 15.0 27.36811065673828 27.36811065673828\n",
      "\tgrad:  4.0 28 16.76190185546875 16.76190185546875\n",
      "\tgrad:  5.0 45 -60.260009765625 -60.260009765625\n",
      "Epoch: 61 | Loss: 1.452507495880127 | w1: 1.7858058214187622 | w2: 0.2059536725282669\n",
      "\tgrad:  1.0 1.0 1.9835190773010254 1.9835190773010254\n",
      "\tgrad:  2.0 6.0 12.326793670654297 12.326793670654297\n",
      "\tgrad:  3.0 15.0 27.112163543701172 27.112163543701172\n",
      "\tgrad:  4.0 28 16.6051025390625 16.6051025390625\n",
      "\tgrad:  5.0 45 -59.696388244628906 -59.696388244628906\n",
      "Epoch: 62 | Loss: 1.4254635572433472 | w1: 1.7878084182739258 | w2: 0.19467811286449432\n",
      "\tgrad:  1.0 1.0 1.964972972869873 1.964972972869873\n",
      "\tgrad:  2.0 6.0 12.211536407470703 12.211536407470703\n",
      "\tgrad:  3.0 15.0 26.858688354492188 26.858688354492188\n",
      "\tgrad:  4.0 28 16.44989013671875 16.44989013671875\n",
      "\tgrad:  5.0 45 -59.13829803466797 -59.13829803466797\n",
      "Epoch: 63 | Loss: 1.398935317993164 | w1: 1.7897924184799194 | w2: 0.1835079789161682\n",
      "\tgrad:  1.0 1.0 1.9466009140014648 1.9466009140014648\n",
      "\tgrad:  2.0 6.0 12.097362518310547 12.097362518310547\n",
      "\tgrad:  3.0 15.0 26.607582092285156 26.607582092285156\n",
      "\tgrad:  4.0 28 16.2960205078125 16.2960205078125\n",
      "\tgrad:  5.0 45 -58.585357666015625 -58.585357666015625\n",
      "Epoch: 64 | Loss: 1.3728976249694824 | w1: 1.7917578220367432 | w2: 0.17244228720664978\n",
      "\tgrad:  1.0 1.0 1.9284002780914307 1.9284002780914307\n",
      "\tgrad:  2.0 6.0 11.984249114990234 11.984249114990234\n",
      "\tgrad:  3.0 15.0 26.358776092529297 26.358776092529297\n",
      "\tgrad:  4.0 28 16.14373779296875 16.14373779296875\n",
      "\tgrad:  5.0 45 -58.037567138671875 -58.037567138671875\n",
      "Epoch: 65 | Loss: 1.3473436832427979 | w1: 1.7937047481536865 | w2: 0.16148003935813904\n",
      "\tgrad:  1.0 1.0 1.910369634628296 1.910369634628296\n",
      "\tgrad:  2.0 6.0 11.872196197509766 11.872196197509766\n",
      "\tgrad:  3.0 15.0 26.1123046875 26.1123046875\n",
      "\tgrad:  4.0 28 15.99273681640625 15.99273681640625\n",
      "\tgrad:  5.0 45 -57.49492645263672 -57.49492645263672\n",
      "Epoch: 66 | Loss: 1.3222665786743164 | w1: 1.7956336736679077 | w2: 0.15062032639980316\n",
      "\tgrad:  1.0 1.0 1.8925080299377441 1.8925080299377441\n",
      "\tgrad:  2.0 6.0 11.761192321777344 11.761192321777344\n",
      "\tgrad:  3.0 15.0 25.868167877197266 25.868167877197266\n",
      "\tgrad:  4.0 28 15.8431396484375 15.8431396484375\n",
      "\tgrad:  5.0 45 -56.957435607910156 -56.957435607910156\n",
      "Epoch: 67 | Loss: 1.2976597547531128 | w1: 1.7975444793701172 | w2: 0.13986217975616455\n",
      "\tgrad:  1.0 1.0 1.8748133182525635 1.8748133182525635\n",
      "\tgrad:  2.0 6.0 11.651229858398438 11.651229858398438\n",
      "\tgrad:  3.0 15.0 25.626331329345703 25.626331329345703\n",
      "\tgrad:  4.0 28 15.69512939453125 15.69512939453125\n",
      "\tgrad:  5.0 45 -56.424903869628906 -56.424903869628906\n",
      "Epoch: 68 | Loss: 1.2735079526901245 | w1: 1.7994375228881836 | w2: 0.12920458614826202\n",
      "\tgrad:  1.0 1.0 1.8572843074798584 1.8572843074798584\n",
      "\tgrad:  2.0 6.0 11.542293548583984 11.542293548583984\n",
      "\tgrad:  3.0 15.0 25.38672637939453 25.38672637939453\n",
      "\tgrad:  4.0 28 15.54833984375 15.54833984375\n",
      "\tgrad:  5.0 45 -55.89733123779297 -55.89733123779297\n",
      "Epoch: 69 | Loss: 1.2498046159744263 | w1: 1.8013126850128174 | w2: 0.11864663660526276\n",
      "\tgrad:  1.0 1.0 1.839918613433838 1.839918613433838\n",
      "\tgrad:  2.0 6.0 11.434371948242188 11.434371948242188\n",
      "\tgrad:  3.0 15.0 25.14935302734375 25.14935302734375\n",
      "\tgrad:  4.0 28 15.4029541015625 15.4029541015625\n",
      "\tgrad:  5.0 45 -55.374717712402344 -55.374717712402344\n",
      "Epoch: 70 | Loss: 1.2265437841415405 | w1: 1.8031704425811768 | w2: 0.10818740725517273\n",
      "\tgrad:  1.0 1.0 1.8227157592773438 1.8227157592773438\n",
      "\tgrad:  2.0 6.0 11.327465057373047 11.327465057373047\n",
      "\tgrad:  3.0 15.0 24.91421127319336 24.91421127319336\n",
      "\tgrad:  4.0 28 15.25897216796875 15.25897216796875\n",
      "\tgrad:  5.0 45 -54.85706329345703 -54.85706329345703\n",
      "Epoch: 71 | Loss: 1.203718900680542 | w1: 1.8050107955932617 | w2: 0.09782599657773972\n",
      "\tgrad:  1.0 1.0 1.805673599243164 1.805673599243164\n",
      "\tgrad:  2.0 6.0 11.221553802490234 11.221553802490234\n",
      "\tgrad:  3.0 15.0 24.68126678466797 24.68126678466797\n",
      "\tgrad:  4.0 28 15.11627197265625 15.11627197265625\n",
      "\tgrad:  5.0 45 -54.34398651123047 -54.34398651123047\n",
      "Epoch: 72 | Loss: 1.1813075542449951 | w1: 1.8068337440490723 | w2: 0.08756142854690552\n",
      "\tgrad:  1.0 1.0 1.788790225982666 1.788790225982666\n",
      "\tgrad:  2.0 6.0 11.116626739501953 11.116626739501953\n",
      "\tgrad:  3.0 15.0 24.450450897216797 24.450450897216797\n",
      "\tgrad:  4.0 28 14.974853515625 14.974853515625\n",
      "\tgrad:  5.0 45 -53.83586883544922 -53.83586883544922\n",
      "Epoch: 73 | Loss: 1.15932035446167 | w1: 1.8086398839950562 | w2: 0.07739287614822388\n",
      "\tgrad:  1.0 1.0 1.7720656394958496 1.7720656394958496\n",
      "\tgrad:  2.0 6.0 11.012691497802734 11.012691497802734\n",
      "\tgrad:  3.0 15.0 24.221900939941406 24.221900939941406\n",
      "\tgrad:  4.0 28 14.83489990234375 14.83489990234375\n",
      "\tgrad:  5.0 45 -53.33251953125 -53.33251953125\n",
      "Epoch: 74 | Loss: 1.1377429962158203 | w1: 1.8104290962219238 | w2: 0.067319355905056\n",
      "\tgrad:  1.0 1.0 1.7554969787597656 1.7554969787597656\n",
      "\tgrad:  2.0 6.0 10.909725189208984 10.909725189208984\n",
      "\tgrad:  3.0 15.0 23.995410919189453 23.995410919189453\n",
      "\tgrad:  4.0 28 14.69622802734375 14.69622802734375\n",
      "\tgrad:  5.0 45 -52.83393859863281 -52.83393859863281\n",
      "Epoch: 75 | Loss: 1.116569995880127 | w1: 1.8122016191482544 | w2: 0.05734003335237503\n",
      "\tgrad:  1.0 1.0 1.7390832901000977 1.7390832901000977\n",
      "\tgrad:  2.0 6.0 10.807720184326172 10.807720184326172\n",
      "\tgrad:  3.0 15.0 23.77104949951172 23.77104949951172\n",
      "\tgrad:  4.0 28 14.558837890625 14.558837890625\n",
      "\tgrad:  5.0 45 -52.339935302734375 -52.339935302734375\n",
      "Epoch: 76 | Loss: 1.095787525177002 | w1: 1.8139575719833374 | w2: 0.04745401814579964\n",
      "\tgrad:  1.0 1.0 1.722823143005371 1.722823143005371\n",
      "\tgrad:  2.0 6.0 10.706668853759766 10.706668853759766\n",
      "\tgrad:  3.0 15.0 23.548816680908203 23.548816680908203\n",
      "\tgrad:  4.0 28 14.42266845703125 14.42266845703125\n",
      "\tgrad:  5.0 45 -51.850318908691406 -51.850318908691406\n",
      "Epoch: 77 | Loss: 1.0753822326660156 | w1: 1.8156967163085938 | w2: 0.0376603789627552\n",
      "\tgrad:  1.0 1.0 1.706714153289795 1.706714153289795\n",
      "\tgrad:  2.0 6.0 10.606555938720703 10.606555938720703\n",
      "\tgrad:  3.0 15.0 23.328609466552734 23.328609466552734\n",
      "\tgrad:  4.0 28 14.28778076171875 14.28778076171875\n",
      "\tgrad:  5.0 45 -51.36585235595703 -51.36585235595703\n",
      "Epoch: 78 | Loss: 1.0553803443908691 | w1: 1.817420244216919 | w2: 0.02795841544866562\n",
      "\tgrad:  1.0 1.0 1.6907572746276855 1.6907572746276855\n",
      "\tgrad:  2.0 6.0 10.507392883300781 10.507392883300781\n",
      "\tgrad:  3.0 15.0 23.110530853271484 23.110530853271484\n",
      "\tgrad:  4.0 28 14.15423583984375 14.15423583984375\n",
      "\tgrad:  5.0 45 -50.88539123535156 -50.88539123535156\n",
      "Epoch: 79 | Loss: 1.035729169845581 | w1: 1.8191272020339966 | w2: 0.018347082659602165\n",
      "\tgrad:  1.0 1.0 1.6749484539031982 1.6749484539031982\n",
      "\tgrad:  2.0 6.0 10.40914535522461 10.40914535522461\n",
      "\tgrad:  3.0 15.0 22.8944091796875 22.8944091796875\n",
      "\tgrad:  4.0 28 14.0218505859375 14.0218505859375\n",
      "\tgrad:  5.0 45 -50.409889221191406 -50.409889221191406\n",
      "Epoch: 80 | Loss: 1.016462802886963 | w1: 1.820818543434143 | w2: 0.008825712837278843\n",
      "\tgrad:  1.0 1.0 1.6592884063720703 1.6592884063720703\n",
      "\tgrad:  2.0 6.0 10.31182861328125 10.31182861328125\n",
      "\tgrad:  3.0 15.0 22.680381774902344 22.680381774902344\n",
      "\tgrad:  4.0 28 13.890869140625 13.890869140625\n",
      "\tgrad:  5.0 45 -49.938392639160156 -49.938392639160156\n",
      "Epoch: 81 | Loss: 0.9975371956825256 | w1: 1.8224937915802002 | w2: -0.0006067296490073204\n",
      "\tgrad:  1.0 1.0 1.6437740325927734 1.6437740325927734\n",
      "\tgrad:  2.0 6.0 10.215412139892578 10.215412139892578\n",
      "\tgrad:  3.0 15.0 22.468311309814453 22.468311309814453\n",
      "\tgrad:  4.0 28 13.76092529296875 13.76092529296875\n",
      "\tgrad:  5.0 45 -49.471473693847656 -49.471473693847656\n",
      "Epoch: 82 | Loss: 0.9789707064628601 | w1: 1.8241534233093262 | w2: -0.009950952604413033\n",
      "\tgrad:  1.0 1.0 1.6284048557281494 1.6284048557281494\n",
      "\tgrad:  2.0 6.0 10.11989974975586 10.11989974975586\n",
      "\tgrad:  3.0 15.0 22.25823211669922 22.25823211669922\n",
      "\tgrad:  4.0 28 13.63226318359375 13.63226318359375\n",
      "\tgrad:  5.0 45 -49.008941650390625 -49.008941650390625\n",
      "Epoch: 83 | Loss: 0.9607505202293396 | w1: 1.8257975578308105 | w2: -0.019207805395126343\n",
      "\tgrad:  1.0 1.0 1.6131794452667236 1.6131794452667236\n",
      "\tgrad:  2.0 6.0 10.025276184082031 10.025276184082031\n",
      "\tgrad:  3.0 15.0 22.05010986328125 22.05010986328125\n",
      "\tgrad:  4.0 28 13.5047607421875 13.5047607421875\n",
      "\tgrad:  5.0 45 -48.55060577392578 -48.55060577392578\n",
      "Epoch: 84 | Loss: 0.9428645372390747 | w1: 1.8274261951446533 | w2: -0.028378114104270935\n",
      "\tgrad:  1.0 1.0 1.5980961322784424 1.5980961322784424\n",
      "\tgrad:  2.0 6.0 9.931537628173828 9.931537628173828\n",
      "\tgrad:  3.0 15.0 21.843910217285156 21.843910217285156\n",
      "\tgrad:  4.0 28 13.37847900390625 13.37847900390625\n",
      "\tgrad:  5.0 45 -48.096656799316406 -48.096656799316406\n",
      "Epoch: 85 | Loss: 0.9253153800964355 | w1: 1.8290398120880127 | w2: -0.03746265918016434\n",
      "\tgrad:  1.0 1.0 1.5831542015075684 1.5831542015075684\n",
      "\tgrad:  2.0 6.0 9.838680267333984 9.838680267333984\n",
      "\tgrad:  3.0 15.0 21.63970184326172 21.63970184326172\n",
      "\tgrad:  4.0 28 13.25341796875 13.25341796875\n",
      "\tgrad:  5.0 45 -47.64690399169922 -47.64690399169922\n",
      "Epoch: 86 | Loss: 0.9080910086631775 | w1: 1.8306381702423096 | w2: -0.046462297439575195\n",
      "\tgrad:  1.0 1.0 1.5683517456054688 1.5683517456054688\n",
      "\tgrad:  2.0 6.0 9.746685028076172 9.746685028076172\n",
      "\tgrad:  3.0 15.0 21.437347412109375 21.437347412109375\n",
      "\tgrad:  4.0 28 13.1295166015625 13.1295166015625\n",
      "\tgrad:  5.0 45 -47.2015380859375 -47.2015380859375\n",
      "Epoch: 87 | Loss: 0.8911941051483154 | w1: 1.8322217464447021 | w2: -0.05537775158882141\n",
      "\tgrad:  1.0 1.0 1.5536880493164062 1.5536880493164062\n",
      "\tgrad:  2.0 6.0 9.655559539794922 9.655559539794922\n",
      "\tgrad:  3.0 15.0 21.236949920654297 21.236949920654297\n",
      "\tgrad:  4.0 28 13.0067138671875 13.0067138671875\n",
      "\tgrad:  5.0 45 -46.76036834716797 -46.76036834716797\n",
      "Epoch: 88 | Loss: 0.8746128082275391 | w1: 1.83379065990448 | w2: -0.06420981138944626\n",
      "\tgrad:  1.0 1.0 1.5391616821289062 1.5391616821289062\n",
      "\tgrad:  2.0 6.0 9.565288543701172 9.565288543701172\n",
      "\tgrad:  3.0 15.0 21.038406372070312 21.038406372070312\n",
      "\tgrad:  4.0 28 12.88519287109375 12.88519287109375\n",
      "\tgrad:  5.0 45 -46.32301330566406 -46.32301330566406\n",
      "Epoch: 89 | Loss: 0.858328640460968 | w1: 1.8353445529937744 | w2: -0.07295937091112137\n",
      "\tgrad:  1.0 1.0 1.5247702598571777 1.5247702598571777\n",
      "\tgrad:  2.0 6.0 9.475849151611328 9.475849151611328\n",
      "\tgrad:  3.0 15.0 20.84164810180664 20.84164810180664\n",
      "\tgrad:  4.0 28 12.7646484375 12.7646484375\n",
      "\tgrad:  5.0 45 -45.889854431152344 -45.889854431152344\n",
      "Epoch: 90 | Loss: 0.842351496219635 | w1: 1.8368840217590332 | w2: -0.08162709325551987\n",
      "\tgrad:  1.0 1.0 1.5105137825012207 1.5105137825012207\n",
      "\tgrad:  2.0 6.0 9.387252807617188 9.387252807617188\n",
      "\tgrad:  3.0 15.0 20.646812438964844 20.646812438964844\n",
      "\tgrad:  4.0 28 12.64532470703125 12.64532470703125\n",
      "\tgrad:  5.0 45 -45.46089172363281 -45.46089172363281\n",
      "Epoch: 91 | Loss: 0.8266770839691162 | w1: 1.838409185409546 | w2: -0.09021376073360443\n",
      "\tgrad:  1.0 1.0 1.4963908195495605 1.4963908195495605\n",
      "\tgrad:  2.0 6.0 9.299480438232422 9.299480438232422\n",
      "\tgrad:  3.0 15.0 20.45376205444336 20.45376205444336\n",
      "\tgrad:  4.0 28 12.527099609375 12.527099609375\n",
      "\tgrad:  5.0 45 -45.03593444824219 -45.03593444824219\n",
      "Epoch: 92 | Loss: 0.8112941384315491 | w1: 1.8399202823638916 | w2: -0.09872013330459595\n",
      "\tgrad:  1.0 1.0 1.4824004173278809 1.4824004173278809\n",
      "\tgrad:  2.0 6.0 9.212543487548828 9.212543487548828\n",
      "\tgrad:  3.0 15.0 20.26259994506836 20.26259994506836\n",
      "\tgrad:  4.0 28 12.41009521484375 12.41009521484375\n",
      "\tgrad:  5.0 45 -44.614601135253906 -44.614601135253906\n",
      "Epoch: 93 | Loss: 0.7961850762367249 | w1: 1.8414167165756226 | w2: -0.10714709758758545\n",
      "\tgrad:  1.0 1.0 1.4685392379760742 1.4685392379760742\n",
      "\tgrad:  2.0 6.0 9.126392364501953 9.126392364501953\n",
      "\tgrad:  3.0 15.0 20.07305145263672 20.07305145263672\n",
      "\tgrad:  4.0 28 12.29388427734375 12.29388427734375\n",
      "\tgrad:  5.0 45 -44.19746398925781 -44.19746398925781\n",
      "Epoch: 94 | Loss: 0.7813663482666016 | w1: 1.8428994417190552 | w2: -0.11549517512321472\n",
      "\tgrad:  1.0 1.0 1.4548084735870361 1.4548084735870361\n",
      "\tgrad:  2.0 6.0 9.041061401367188 9.041061401367188\n",
      "\tgrad:  3.0 15.0 19.885391235351562 19.885391235351562\n",
      "\tgrad:  4.0 28 12.178955078125 12.178955078125\n",
      "\tgrad:  5.0 45 -43.784332275390625 -43.784332275390625\n",
      "Epoch: 95 | Loss: 0.7668271064758301 | w1: 1.8443684577941895 | w2: -0.12376518547534943\n",
      "\tgrad:  1.0 1.0 1.441206455230713 1.441206455230713\n",
      "\tgrad:  2.0 6.0 8.956531524658203 8.956531524658203\n",
      "\tgrad:  3.0 15.0 19.699481964111328 19.699481964111328\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tgrad:  4.0 28 12.065185546875 12.065185546875\n",
      "\tgrad:  5.0 45 -43.37482452392578 -43.37482452392578\n",
      "Epoch: 96 | Loss: 0.7525501847267151 | w1: 1.8458234071731567 | w2: -0.13195794820785522\n",
      "\tgrad:  1.0 1.0 1.4277310371398926 1.4277310371398926\n",
      "\tgrad:  2.0 6.0 8.872783660888672 8.872783660888672\n",
      "\tgrad:  3.0 15.0 19.515254974365234 19.515254974365234\n",
      "\tgrad:  4.0 28 11.9522705078125 11.9522705078125\n",
      "\tgrad:  5.0 45 -42.969322204589844 -42.969322204589844\n",
      "Epoch: 97 | Loss: 0.7385450601577759 | w1: 1.8472650051116943 | w2: -0.14007404446601868\n",
      "\tgrad:  1.0 1.0 1.414381980895996 1.414381980895996\n",
      "\tgrad:  2.0 6.0 8.789825439453125 8.789825439453125\n",
      "\tgrad:  3.0 15.0 19.332778930664062 19.332778930664062\n",
      "\tgrad:  4.0 28 11.84051513671875 11.84051513671875\n",
      "\tgrad:  5.0 45 -42.56763458251953 -42.56763458251953\n",
      "Epoch: 98 | Loss: 0.7248014211654663 | w1: 1.8486931324005127 | w2: -0.14811421930789948\n",
      "\tgrad:  1.0 1.0 1.4011578559875488 1.4011578559875488\n",
      "\tgrad:  2.0 6.0 8.707645416259766 8.707645416259766\n",
      "\tgrad:  3.0 15.0 19.152019500732422 19.152019500732422\n",
      "\tgrad:  4.0 28 11.72979736328125 11.72979736328125\n",
      "\tgrad:  5.0 45 -42.169761657714844 -42.169761657714844\n",
      "Epoch: 99 | Loss: 0.7113155126571655 | w1: 1.850108027458191 | w2: -0.15607918798923492\n",
      "Predict (After training) 6 65.66741180419922\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "\n",
    "print('Predict (before training)', 6, quad_forward(6).item())\n",
    "\n",
    "for epoch in range(100):\n",
    "    for x_val, y_val in zip(x_data, y_data):\n",
    "        y_pred = quad_forward(x_val)\n",
    "        l = loss(y_pred, y_val)\n",
    "        l.backward()\n",
    "        print(\"\\tgrad: \", x_val, y_val, w_1.grad.item(), w_1.grad.item())\n",
    "        w_1.data = w_1.data - 0.0012*w_1.grad.item()\n",
    "        w_2.data = w_2.data - 0.0012*w_2.grad.item()\n",
    "        \n",
    "        # Manually zero the gradients after updating weights\n",
    "        w_1.grad.data.zero_()\n",
    "        w_2.grad.data.zero_()\n",
    "        \n",
    "    print(f\"Epoch: {epoch} | Loss: {l.item()} | w1: {w_1.item()} | w2: {w_2.item()}\")\n",
    "\n",
    "\n",
    "print('Predict (After training)', 6, quad_forward(6).item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}