GEOLoaderFinal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bfe2d085",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import warnings\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch import optim\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from torch.autograd import Variable\n",
    "import torch.utils.data as data_utils\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from utils.losses import TiltedLC\n",
    "from utils.loops import trainCLRregression, trainLRregression, trainLBFGS\n",
    "from utils.networks import regressionNetwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7df51650",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def GEOdataLoader(path, bSize= 256):\n",
    "    \"\"\"\n",
    "    Specify the path to the parent folder containing the .npy files\n",
    "    \"\"\"\n",
    "    X_tr= np.load(os.path.join(path, \"X_tr.npy\"))\n",
    "    Y_tr= np.load(os.path.join(path, \"Y_tr.npy\"))\n",
    "    X_va= np.load(os.path.join(path, \"X_va.npy\"))\n",
    "    Y_va= np.load(os.path.join(path, \"Y_va.npy\"))\n",
    "\n",
    "    test_data= torch.utils.data.TensorDataset(torch.from_numpy(X_tr).float(), torch.from_numpy(Y_tr).float())\n",
    "    val_data= torch.utils.data.TensorDataset(torch.from_numpy(X_va).float(), torch.from_numpy(Y_va).float())\n",
    "    trainLoader= torch.utils.data.DataLoader(test_data, batch_size=bSize, shuffle=True) \n",
    "    valLoader= torch.utils.data.DataLoader(val_data, batch_size=bSize, shuffle=True) \n",
    "    # trainLoaderLBFGS= torch.utils.data.DataLoader(test_data, batch_size= int(X_tr.shape[0]/7), shuffle=True) \n",
    "    return trainLoader, valLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3a61466f",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# global initialisations:\n",
    "h= 0.4 # smoothing parameter for the log-cosh \n",
    "tau= 0.5\n",
    "device= ('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "trainLoader, testLoader= GEOdataLoader(\"/home/aryamanj/Downloads/LGdata\")\n",
    "criterion1= TiltedLC()\n",
    "criterion2= nn.L1Loss()\n",
    "criterion3= nn.MSELoss()\n",
    "N_EPOCHS= 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c81afb1d",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# instantiating objects for all constantLR tests:\n",
    "\n",
    "size1,size2= 1000, 1000\n",
    "indim, outdim= 943, 4760\n",
    "# LC initialisations\n",
    "model_CLR_LC= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_CLR_LC= optim.Adam(model_CLR_LC.parameters(), lr= 0.01)\n",
    "lossList_CLR_LC= []\n",
    "valList_CLR_LC= []\n",
    "model_LALR_LC= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_LALR_LC= optim.Adam(model_LALR_LC.parameters(), lr= 0.01)\n",
    "lossList_LALR_LC= []\n",
    "valList_LALR_LC= []\n",
    "model_LBFGS_LC= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_LBFGS_LC= optim.Adam(model_LBFGS_LC.parameters(), lr= 0.01)\n",
    "lossList_LBFGS_LC= []\n",
    "valList_LBFGS_LC= []\n",
    "\n",
    "# L1 initialisations\n",
    "model_CLR_L1= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_CLR_L1= optim.Adam(model_CLR_L1.parameters(), lr= 0.01)\n",
    "lossList_CLR_L1= []\n",
    "valList_CLR_L1= []\n",
    "model_LALR_L1= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_LALR_L1= optim.Adam(model_LALR_L1.parameters(), lr= 0.01)\n",
    "lossList_LALR_L1= []\n",
    "valList_LALR_L1= []\n",
    "\n",
    "# MSE initialisations:\n",
    "model_CLR_MSE= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_CLR_MSE= optim.Adam(model_CLR_MSE.parameters(), lr= 0.01)\n",
    "lossList_CLR_MSE= []\n",
    "valList_CLR_MSE= []\n",
    "model_LALR_MSE= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_LALR_MSE= optim.Adam(model_LALR_MSE.parameters(), lr= 0.01)\n",
    "lossList_LALR_MSE= []\n",
    "valList_LALR_MSE= []\n",
    "model_LBFGS_MSE= regressionNetwork(size1, size2, indim, outdim, 0.1).to(device)\n",
    "optimizer_LBFGS_MSE= optim.Adam(model_LBFGS_MSE.parameters(), lr= 0.01)\n",
    "lossList_LBFGS_MSE= []\n",
    "valList_LBFGS_MSE= []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da671fa5",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ConstantLR training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "fd80613e",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training loss: 0.13340913362650775 Validation loss: 0.7101643735712225\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training loss: 0.09001756015016298 Validation loss: 0.6816301061348482\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training loss: 0.08415809314515474 Validation loss: 0.6762900907884944\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training loss: 0.08295023707560228 Validation loss: 0.6748038083314896\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training loss: 0.08346389936181242 Validation loss: 0.6757863434878263\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training loss: 0.08501505783037082 Validation loss: 0.6801524853164499\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training loss: 0.08682899597914143 Validation loss: 0.6858692534945228\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training loss: 0.08884543463888361 Validation loss: 0.6894284838979895\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training loss: 0.09074627243046801 Validation loss: 0.697966761209748\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training loss: 0.09226981440950884 Validation loss: 0.6969334415414117\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training loss: 0.093534363385408 Validation loss: 0.701411550695246\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training loss: 0.09427339712463813 Validation loss: 0.7008837041529742\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training loss: 0.09499254382567035 Validation loss: 0.7053757269274105\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training loss: 0.09544030179723195 Validation loss: 0.7048382718454708\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training loss: 0.09557137256139293 Validation loss: 0.7054269991137765\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training loss: 0.09588280747448677 Validation loss: 0.7072272747755051\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training loss: 0.09621431584059333 Validation loss: 0.7041001103141091\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training loss: 0.09643569481304125 Validation loss: 0.7052571773529053\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training loss: 0.09629892918579173 Validation loss: 0.7057977386496284\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training loss: 0.09647089616537781 Validation loss: 0.7052430781451139\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training loss: 0.09660422900739939 Validation loss: 0.7059973099014976\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training loss: 0.09643465843083879 Validation loss: 0.7093886909159747\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training loss: 0.09663236005934926 Validation loss: 0.7091420008377596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training loss: 0.0964912748989523 Validation loss: 0.7088119360533628\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training loss: 0.09662807694171965 Validation loss: 0.7090961540287192\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training loss: 0.09669142072585543 Validation loss: 0.708813336762515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training loss: 0.09647496074695752 Validation loss: 0.7061544453555887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training loss: 0.0966641304944709 Validation loss: 0.7072393460707231\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training loss: 0.09647365597208227 Validation loss: 0.710883850401098\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training loss: 0.09648303118580014 Validation loss: 0.7080794979225505\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training loss: 0.09651696143043831 Validation loss: 0.7094827334989201\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training loss: 0.09644054198471201 Validation loss: 0.7063934708183462\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training loss: 0.09659957572259546 Validation loss: 0.7077071206136183\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training loss: 0.09682144044283832 Validation loss: 0.7097713080319491\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training loss: 0.09663524850129394 Validation loss: 0.7070901434529911\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training loss: 0.0963208918440926 Validation loss: 0.708367107944055\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training loss: 0.09663027920022134 Validation loss: 0.7088593149727042\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training loss: 0.09638421151583064 Validation loss: 0.7073995538733222\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training loss: 0.09648622349119323 Validation loss: 0.7082902558825233\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training loss: 0.09644862906390034 Validation loss: 0.7064830416982825\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training loss: 0.09669920282508179 Validation loss: 0.7077529023994099\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training loss: 0.09663514765006322 Validation loss: 0.7090880816633051\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training loss: 0.09671227947435736 Validation loss: 0.7093202783302828\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training loss: 0.09656072330663802 Validation loss: 0.7057999237017198\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training loss: 0.09662508702587326 Validation loss: 0.7085399424487894\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training loss: 0.09674048421619949 Validation loss: 0.7073089277202432\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training loss: 0.09651564875408276 Validation loss: 0.7106554101813923\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training loss: 0.09669204734759647 Validation loss: 0.707237499681386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training loss: 0.0965510075562969 Validation loss: 0.7105454071001573\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training loss: 0.09666727607806752 Validation loss: 0.7059584571556612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training loss: 0.09647934168834851 Validation loss: 0.7075310769406232\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training loss: 0.0966971088039772 Validation loss: 0.7078688185323369\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training loss: 0.09674571357903632 Validation loss: 0.7072879658503965\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training loss: 0.09656587380497188 Validation loss: 0.7070103734731674\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training loss: 0.09678166909331204 Validation loss: 0.7097566032951529\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training loss: 0.09658204768422014 Validation loss: 0.7063497440381483\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training loss: 0.0965852860243245 Validation loss: 0.7072678966955706\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training loss: 0.09643000654167676 Validation loss: 0.7097486812960018\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training loss: 0.09637334519830835 Validation loss: 0.7060928764668378\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training loss: 0.0965237563599427 Validation loss: 0.7092593962495978\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training loss: 0.09657844847406365 Validation loss: 0.7071623260324652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training loss: 0.09628203060338408 Validation loss: 0.7087181088599291\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training loss: 0.09647603081780827 Validation loss: 0.7084579508412968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training loss: 0.09665911395628789 Validation loss: 0.7074195065281608\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training loss: 0.09667856334265783 Validation loss: 0.7069882994348352\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training loss: 0.09651786084649198 Validation loss: 0.7088096114722165\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training loss: 0.09643417613994147 Validation loss: 0.7091420116749677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training loss: 0.0963612327872848 Validation loss: 0.7077840837565336\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training loss: 0.09657777719178186 Validation loss: 0.7073842205784537\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training loss: 0.09647572281037696 Validation loss: 0.7083706219087947\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training loss: 0.09661251014522586 Validation loss: 0.7107062434608286\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training loss: 0.09658018965542488 Validation loss: 0.7088905369693582\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training loss: 0.09640378082004679 Validation loss: 0.7085552608424966\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training loss: 0.0964446203509394 Validation loss: 0.7102197232571515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training loss: 0.09648843257760452 Validation loss: 0.7046226899732243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training loss: 0.09642175376758795 Validation loss: 0.7081862132657658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training loss: 0.09637035234087826 Validation loss: 0.706778579137542\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training loss: 0.09650364668465485 Validation loss: 0.7069274281913583\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training loss: 0.09642943082865789 Validation loss: 0.7113216695460406\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training loss: 0.09648784694479247 Validation loss: 0.7098547491160306\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training loss: 0.09659320496087115 Validation loss: 0.7099304253404791\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training loss: 0.09645617313103305 Validation loss: 0.7104464986107566\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training loss: 0.0964337220195391 Validation loss: 0.7123006067492745\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training loss: 0.09658075321562352 Validation loss: 0.7079767679626291\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training loss: 0.09634204917407517 Validation loss: 0.7066305631941016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training loss: 0.09659647866439407 Validation loss: 0.7084064686840231\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training loss: 0.09634713596672421 Validation loss: 0.7081871777772903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training loss: 0.09643065843527186 Validation loss: 0.7054010236805136\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training loss: 0.09655235647639898 Validation loss: 0.7098237289623781\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training loss: 0.09643120889790807 Validation loss: 0.7080773088065061\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training loss: 0.09647736262519353 Validation loss: 0.7062382603233511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training loss: 0.09638990328703559 Validation loss: 0.7073373198509216\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training loss: 0.09648884871507585 Validation loss: 0.7056817656213586\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training loss: 0.09635462605472257 Validation loss: 0.7073036960580132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training loss: 0.09651995692885232 Validation loss: 0.7077340131456201\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training loss: 0.0962211006355904 Validation loss: 0.7058817906813188\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training loss: 0.09629857145056601 Validation loss: 0.7084705477411096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training loss: 0.0965266477864139 Validation loss: 0.7066934257745743\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training loss: 0.09644520269862512 Validation loss: 0.7086467011408373\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training loss: 0.09654820877897637 Validation loss: 0.709889670664614\n"
     ]
    }
   ],
   "source": [
    "trainCLRregression(model_CLR_LC, optimizer_CLR_LC, trainLoader, testLoader, criterion1, tau, N_EPOCHS, lossList_CLR_LC, valList_CLR_LC, \"LC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "4ec33e2c",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training loss: 0.622708001123038 Validation loss: 0.7061615816571496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training loss: 0.49856454099633163 Validation loss: 0.6859673654491251\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training loss: 0.4919781775914283 Validation loss: 0.6897864246910269\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training loss: 0.4968662262821747 Validation loss: 0.6934022849256342\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training loss: 0.5054506783183095 Validation loss: 0.7001596946607936\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training loss: 0.5144236856475687 Validation loss: 0.7043532539497722\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training loss: 0.5231321811332483 Validation loss: 0.7111925956877795\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training loss: 0.5308830251954818 Validation loss: 0.7175268137996847\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training loss: 0.5365349662406987 Validation loss: 0.7189975678920746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training loss: 0.5408105511830245 Validation loss: 0.7237637422301553\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training loss: 0.5438822913238569 Validation loss: 0.7263004386966879\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training loss: 0.5466594550039308 Validation loss: 0.7285342826084658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training loss: 0.5480013316234182 Validation loss: 0.7269483005458658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training loss: 0.5490905046463013 Validation loss: 0.7269433316859332\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training loss: 0.5499847904062408 Validation loss: 0.7281848679889332\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training loss: 0.5501956000108197 Validation loss: 0.7284707941792228\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training loss: 0.5509598306688864 Validation loss: 0.7281388098543341\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training loss: 0.5505133221403666 Validation loss: 0.7297978455370123\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training loss: 0.5507522498496328 Validation loss: 0.7278693182901903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training loss: 0.5512724359028621 Validation loss: 0.7258051484823227\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training loss: 0.5507392620490676 Validation loss: 0.7297597378492355\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training loss: 0.551262406347808 Validation loss: 0.7293490076606924\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training loss: 0.5510354084996051 Validation loss: 0.726609780029817\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training loss: 0.5503605987221776 Validation loss: 0.7313395806334235\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training loss: 0.5510425814977641 Validation loss: 0.728975239125165\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training loss: 0.5513626355946236 Validation loss: 0.7249960587783293\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training loss: 0.5506162232899184 Validation loss: 0.726635521108454\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training loss: 0.5510787568793173 Validation loss: 0.7273909476670352\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training loss: 0.5503718555832459 Validation loss: 0.7274376546794717\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training loss: 0.5507574488518905 Validation loss: 0.7266508313742551\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training loss: 0.5508575410252002 Validation loss: 0.7296216081489216\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training loss: 0.5507818424392502 Validation loss: 0.7308218872005289\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training loss: 0.5505414560480145 Validation loss: 0.7289864867925644\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training loss: 0.5501617576272069 Validation loss: 0.7263572419231589\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training loss: 0.5503952618291124 Validation loss: 0.7259228771383112\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training loss: 0.5515379891958979 Validation loss: 0.7280726893381639\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training loss: 0.5499699213662821 Validation loss: 0.7267759225585244\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training loss: 0.5508415690416562 Validation loss: 0.7253037067976865\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training loss: 0.5502366552435356 Validation loss: 0.7282298329201612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training loss: 0.5503537163610761 Validation loss: 0.7270427413962104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training loss: 0.550211077121218 Validation loss: 0.7270365127108314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training loss: 0.5503664837790497 Validation loss: 0.7256331687623804\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training loss: 0.550564001719615 Validation loss: 0.728562439029867\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training loss: 0.5501338569849987 Validation loss: 0.7283983636986125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training loss: 0.5504066634933955 Validation loss: 0.728394782001322\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training loss: 0.5506598312160811 Validation loss: 0.7285165922208265\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training loss: 0.5509043387446005 Validation loss: 0.7291399375958876\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training loss: 0.550857587403454 Validation loss: 0.7263618924401023\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training loss: 0.5504657086790123 Validation loss: 0.7297935391014273\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training loss: 0.5508868675410575 Validation loss: 0.7264616232026707\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training loss: 0.5502793891972698 Validation loss: 0.7285261045802723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training loss: 0.5508479680382896 Validation loss: 0.7308079288764433\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training loss: 0.5500648456279413 Validation loss: 0.7307570021260869\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training loss: 0.549771670133992 Validation loss: 0.7262942655520006\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training loss: 0.5500045192001188 Validation loss: 0.7262091474099592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training loss: 0.5502827528230738 Validation loss: 0.732793622396209\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training loss: 0.5506834691814455 Validation loss: 0.7285295372659509\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training loss: 0.550116819504015 Validation loss: 0.7265408323569731\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training loss: 0.5499852581395196 Validation loss: 0.7256185574965044\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training loss: 0.5499913168571868 Validation loss: 0.7275528420101512\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training loss: 0.5499041013484043 Validation loss: 0.7313853434541009\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training loss: 0.5503196542819571 Validation loss: 0.7268826853145253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training loss: 0.5499101334415183 Validation loss: 0.7303903089328245\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training loss: 0.5504558007380461 Validation loss: 0.729894754561511\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training loss: 0.5497261333877828 Validation loss: 0.7254303287376057\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training loss: 0.5503399337746568 Validation loss: 0.727504707195542\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training loss: 0.5498104432130753 Validation loss: 0.728067854588682\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training loss: 0.5502689343364506 Validation loss: 0.7261326516216452\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training loss: 0.5507473318652392 Validation loss: 0.7286818406798623\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training loss: 0.5504071793226413 Validation loss: 0.725035313855518\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training loss: 0.5508420197695751 Validation loss: 0.7297485714608972\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training loss: 0.5505322592402742 Validation loss: 0.728898824615912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training loss: 0.5506835761949721 Validation loss: 0.7285221909934824\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training loss: 0.55011438498923 Validation loss: 0.7254614572633397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training loss: 0.550215241029558 Validation loss: 0.7247625657103278\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training loss: 0.5508990227660803 Validation loss: 0.7308187213811007\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training loss: 0.5504823459328423 Validation loss: 0.7254667918790471\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training loss: 0.5500307984929264 Validation loss: 0.7299646816470406\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training loss: 0.5502418101005664 Validation loss: 0.727906347675757\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training loss: 0.5500060503695815 Validation loss: 0.7278669611974196\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training loss: 0.5504050486713047 Validation loss: 0.7298262972723354\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training loss: 0.549344985388885 Validation loss: 0.7290405414321206\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training loss: 0.5499100903269193 Validation loss: 0.7289841175079346\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training loss: 0.550511625039818 Validation loss: 0.7288581905039874\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training loss: 0.5504960988028249 Validation loss: 0.7291300974108956\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training loss: 0.5496068911181403 Validation loss: 0.7272197861563076\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training loss: 0.5502805718427433 Validation loss: 0.7268781458789652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training loss: 0.550446090849401 Validation loss: 0.7273416668176651\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training loss: 0.5505086985719994 Validation loss: 0.7275157123804092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training loss: 0.5500708629487228 Validation loss: 0.7271716933358799\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training loss: 0.5500304214548996 Validation loss: 0.7267196720296686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training loss: 0.5502624305593178 Validation loss: 0.728491642258384\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training loss: 0.5502819649080721 Validation loss: 0.7292816042900085\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training loss: 0.5504452976094887 Validation loss: 0.7238066426732324\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training loss: 0.5499556023380598 Validation loss: 0.7253424498167905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training loss: 0.5500166172252952 Validation loss: 0.7300235845825889\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training loss: 0.5503023953877539 Validation loss: 0.728867228735577\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training loss: 0.5500892632632847 Validation loss: 0.7267425561493094\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training loss: 0.5492174728459515 Validation loss: 0.7292414605617523\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training loss: 0.5497141147209519 Validation loss: 0.726790273731405\n"
     ]
    }
   ],
   "source": [
    "trainCLRregression(model_CLR_L1, optimizer_CLR_L1, trainLoader, testLoader, criterion2, tau, N_EPOCHS, lossList_CLR_L1, valList_CLR_L1, \"L1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "85624c38",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training loss: 0.787007353147787 Validation loss: 0.7250110195441679\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training loss: 0.5104744523677771 Validation loss: 0.699372870000926\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training loss: 0.46418376563261843 Validation loss: 0.6859994043003429\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training loss: 0.4402478002333847 Validation loss: 0.6766953779892488\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training loss: 0.42745039356545 Validation loss: 0.6784380782734264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training loss: 0.423858594739815 Validation loss: 0.6765128387646242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training loss: 0.4258277130230016 Validation loss: 0.680455667051402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training loss: 0.4294678885758095 Validation loss: 0.6808965084227648\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training loss: 0.43592771621533705 Validation loss: 0.6831044161861594\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training loss: 0.4455816506995935 Validation loss: 0.6921411915258928\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training loss: 0.4561106286062631 Validation loss: 0.6927691305225546\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training loss: 0.46371954524894954 Validation loss: 0.6975717544555664\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training loss: 0.4733807029744734 Validation loss: 0.7019363113424995\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training loss: 0.47975894748649267 Validation loss: 0.7031614523042332\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training loss: 0.4854780709537374 Validation loss: 0.7072966654192318\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training loss: 0.48926310211162405 Validation loss: 0.7058342979712919\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training loss: 0.4925850496683753 Validation loss: 0.7082284187728708\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training loss: 0.49393743688847214 Validation loss: 0.7088259851390665\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training loss: 0.49616743654270334 Validation loss: 0.7082264193079688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training loss: 0.49773977486475746 Validation loss: 0.709494798020883\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training loss: 0.5000890156721176 Validation loss: 0.7121294195001776\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training loss: 0.49999886854237713 Validation loss: 0.7102862217209556\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training loss: 0.5011978568536061 Validation loss: 0.7113652987913652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training loss: 0.5034595131874084 Validation loss: 0.7119516838680614\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training loss: 0.5021958141574255 Validation loss: 0.7125562375242059\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training loss: 0.5026571885481348 Validation loss: 0.712851188399575\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training loss: 0.5027348412045141 Validation loss: 0.7120324034582485\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training loss: 0.5024812792666707 Validation loss: 0.7145664962855253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training loss: 0.5026191801090405 Validation loss: 0.7119286846030842\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training loss: 0.503862413961537 Validation loss: 0.7133331515572288\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training loss: 0.5029414785698443 Validation loss: 0.7128842175006866\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training loss: 0.5023925133500388 Validation loss: 0.7094079703092575\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training loss: 0.5029677420596912 Validation loss: 0.7133320353247903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training loss: 0.5037348831249588 Validation loss: 0.7104982151226564\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training loss: 0.5028242832130245 Validation loss: 0.7112906751307574\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training loss: 0.5028626799411664 Validation loss: 0.7135223692113702\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training loss: 0.5031035283972276 Validation loss: 0.7132242863828485\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training loss: 0.5038647159547558 Validation loss: 0.7121469419110905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training loss: 0.5038225713483881 Validation loss: 0.7117468633434989\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training loss: 0.5033776294601067 Validation loss: 0.7141570760445162\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training loss: 0.5041885609585545 Validation loss: 0.7132246521386233\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training loss: 0.5039829069839774 Validation loss: 0.7110370790416544\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training loss: 0.5033939229136585 Validation loss: 0.7115170020948757\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training loss: 0.5035374118203045 Validation loss: 0.7119117230176926\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training loss: 0.5036023358790263 Validation loss: 0.7112746929580515\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training loss: 0.5024587632256213 Validation loss: 0.7115596830844879\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training loss: 0.5033210567507346 Validation loss: 0.7147247615185651\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training loss: 0.5035173257421004 Validation loss: 0.712903999469497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training loss: 0.5039416244978177 Validation loss: 0.7128195302052931\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training loss: 0.5038513067991658 Validation loss: 0.7126025543971495\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training loss: 0.5031558483066064 Validation loss: 0.7126666009426117\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training loss: 0.502847251442049 Validation loss: 0.7141435539180582\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training loss: 0.5040246732640336 Validation loss: 0.7127031128514897\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training loss: 0.5028851496898475 Validation loss: 0.7143231996081092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training loss: 0.5029329698779741 Validation loss: 0.7117072398012335\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training loss: 0.5031376575530434 Validation loss: 0.7133887843652205\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training loss: 0.5025725439405578 Validation loss: 0.7110554738478227\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training loss: 0.5032335069749816 Validation loss: 0.7115020155906677\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training loss: 0.5034041725936472 Validation loss: 0.7140505042943087\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training loss: 0.5038944631385528 Validation loss: 0.7119888107885014\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training loss: 0.5044725369823082 Validation loss: 0.712388118559664\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training loss: 0.5032449985615458 Validation loss: 0.7144900140437213\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training loss: 0.5029034971847315 Validation loss: 0.7112006233497099\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training loss: 0.5029134795885608 Validation loss: 0.7110293954610825\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training loss: 0.5027001859131739 Validation loss: 0.7137399044903842\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training loss: 0.5034371153250208 Validation loss: 0.7127502804452722\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training loss: 0.5035919105628726 Validation loss: 0.713440862568942\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training loss: 0.5027419284887891 Validation loss: 0.7142864668911154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training loss: 0.5033873838375212 Validation loss: 0.7123480777848851\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training loss: 0.5037150131350635 Validation loss: 0.7137109569527886\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training loss: 0.5019243524466193 Validation loss: 0.7148047062483701\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training loss: 0.5035167139785778 Validation loss: 0.7142161442474886\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training loss: 0.5024227808291356 Validation loss: 0.7125382843342695\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training loss: 0.5020357302526919 Validation loss: 0.7118107798424634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training loss: 0.5041279247411733 Validation loss: 0.7095840492031791\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training loss: 0.5026131627023736 Validation loss: 0.7137708880684592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training loss: 0.5037230673200459 Validation loss: 0.71403222869743\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training loss: 0.5023398978222344 Validation loss: 0.7125404138456691\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training loss: 0.5028288149696262 Validation loss: 0.7109569446607069\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training loss: 0.502522399944256 Validation loss: 0.7126761607148431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training loss: 0.5028156214729166 Validation loss: 0.7112997052344409\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training loss: 0.5027653395442523 Validation loss: 0.7126921320503409\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training loss: 0.5036908583098255 Validation loss: 0.7115065089680932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training loss: 0.5030633132121062 Validation loss: 0.7103628773580898\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training loss: 0.5024605378809167 Validation loss: 0.7138176993890242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training loss: 0.5034912761075352 Validation loss: 0.7116072123700922\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training loss: 0.5032601890715124 Validation loss: 0.7156686349348589\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training loss: 0.5038455743611031 Validation loss: 0.7120915840972554\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training loss: 0.5029960024769093 Validation loss: 0.7139606435190547\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training loss: 0.5027428465205586 Validation loss: 0.7111812870610844\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training loss: 0.5032805485237573 Validation loss: 0.7135312841697172\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training loss: 0.5033579503081373 Validation loss: 0.7130639973011884\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training loss: 0.5038374239841867 Validation loss: 0.7094047489491376\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training loss: 0.5030815198545153 Validation loss: 0.7134177969260649\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training loss: 0.5038959999764687 Validation loss: 0.7127396112138574\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training loss: 0.5022002003378415 Validation loss: 0.7109070867300034\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training loss: 0.5023775007092643 Validation loss: 0.7110817825252359\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training loss: 0.50331602788796 Validation loss: 0.7109071747823195\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training loss: 0.5023970679522248 Validation loss: 0.7134689295833762\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training loss: 0.502907903634162 Validation loss: 0.7119684422558005\n"
     ]
    }
   ],
   "source": [
    "trainCLRregression(model_CLR_MSE, optimizer_CLR_MSE, trainLoader, testLoader, criterion3, tau, N_EPOCHS, lossList_CLR_MSE, valList_CLR_MSE, \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50ec1c48",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# LALR training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5cf6c680",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training Loss: 0.14561908299194634 Validation loss: 0.7210805524479259 LR: 0.012673999182879925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training Loss: 0.08522717613909361 Validation loss: 0.6703318953514099 LR: 0.0032095813658088446\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training Loss: 0.08154976294947633 Validation loss: 0.6633833118460395 LR: 0.003211645409464836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training Loss: 0.07959300028401768 Validation loss: 0.6590548753738403 LR: 0.00321735511533916\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training Loss: 0.07808665430940881 Validation loss: 0.6553888835690238 LR: 0.0032238252460956573\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training Loss: 0.07679037966628582 Validation loss: 0.650961538607424 LR: 0.003229418769478798\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training Loss: 0.07577322824973538 Validation loss: 0.6484643031250347 LR: 0.0032356735318899155\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training Loss: 0.07486242169691437 Validation loss: 0.6482662016695196 LR: 0.0032442640513181686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training Loss: 0.07401644380532356 Validation loss: 0.6466627418994904 LR: 0.0032512599136680365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training Loss: 0.07339312692282866 Validation loss: 0.6442760906436227 LR: 0.0032575067598372698\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training Loss: 0.07286459883884326 Validation loss: 0.6430307003584775 LR: 0.0032640446443110704\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training Loss: 0.07246622956365949 Validation loss: 0.6412537070837888 LR: 0.0032695860136300325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training Loss: 0.07209472128482648 Validation loss: 0.6403663375160911 LR: 0.0032740856986492872\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training Loss: 0.07183454681713917 Validation loss: 0.6415960219773379 LR: 0.003279870143160224\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training Loss: 0.07158089330371588 Validation loss: 0.6406654742631045 LR: 0.003283061785623431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training Loss: 0.07150874787173972 Validation loss: 0.6396262158047069 LR: 0.003284719306975603\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training Loss: 0.07132034701555538 Validation loss: 0.6398178488016129 LR: 0.0032881980296224356\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training Loss: 0.07121039349424736 Validation loss: 0.6392585391348059 LR: 0.003291412489488721\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training Loss: 0.07112372159872 Validation loss: 0.6399133964018389 LR: 0.0032911058515310287\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training Loss: 0.07100266076474093 Validation loss: 0.6385883025147698 LR: 0.0032936378847807646\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training Loss: 0.07095986862347517 Validation loss: 0.6386823532256213 LR: 0.0032944472040981054\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training Loss: 0.0709002504946519 Validation loss: 0.638292824680155 LR: 0.0032940388191491365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training Loss: 0.07086709348930406 Validation loss: 0.6385714452375065 LR: 0.0032953701447695494\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training Loss: 0.07079187572345953 Validation loss: 0.6379967887293209 LR: 0.0032955266069620848\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training Loss: 0.07078106314888605 Validation loss: 0.6384382505308498 LR: 0.003295907750725746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training Loss: 0.07073007255620846 Validation loss: 0.638231175866994 LR: 0.003296544775366783\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training Loss: 0.07074552938685637 Validation loss: 0.6382957818833265 LR: 0.0032971021719276905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training Loss: 0.07069029165190303 Validation loss: 0.6379554732279344 LR: 0.003297863993793726\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training Loss: 0.0706371471650834 Validation loss: 0.6372814706780694 LR: 0.0032975280191749334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training Loss: 0.07061835756305315 Validation loss: 0.6380854939872568 LR: 0.003297313814982772\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training Loss: 0.07062004712713554 Validation loss: 0.6379121934825723 LR: 0.00329763675108552\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training Loss: 0.07057655791642 Validation loss: 0.6386257030747153 LR: 0.003297484014183283\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training Loss: 0.07057763921682017 Validation loss: 0.6378135816617445 LR: 0.0032979592215269804\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training Loss: 0.07055495531225754 Validation loss: 0.6377282386476343 LR: 0.0032980043906718493\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training Loss: 0.07053598492092289 Validation loss: 0.6381856934590773 LR: 0.0032988605089485645\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training Loss: 0.07049103523537474 Validation loss: 0.6382521241903305 LR: 0.0032980297692120075\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training Loss: 0.07050792642088033 Validation loss: 0.6379249597137625 LR: 0.003298311261460185\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training Loss: 0.07050726308302165 Validation loss: 0.6378782391548157 LR: 0.003298953641206026\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training Loss: 0.07050000791376193 Validation loss: 0.638223027641123 LR: 0.0032987359445542097\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training Loss: 0.07048138810767221 Validation loss: 0.6369641450318423 LR: 0.0032981433905661106\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training Loss: 0.07044359444713043 Validation loss: 0.6369803574952212 LR: 0.003297807415947318\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training Loss: 0.07041229563161344 Validation loss: 0.6369185501878912 LR: 0.003298542695119977\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training Loss: 0.07043030264355264 Validation loss: 0.6371424116871574 LR: 0.003298059105873108\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training Loss: 0.07041314924914142 Validation loss: 0.6378286155787382 LR: 0.0032972681801766157\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training Loss: 0.07039931843792671 Validation loss: 0.6384041769938036 LR: 0.003297770395874977\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training Loss: 0.07037147851687688 Validation loss: 0.6376656185496937 LR: 0.0032974195200949907\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training Loss: 0.0703796637848406 Validation loss: 0.6393977904861624 LR: 0.0032974600326269865\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training Loss: 0.07040029446398491 Validation loss: 0.6378650191155347 LR: 0.003297799499705434\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training Loss: 0.07035860960353005 Validation loss: 0.6376818025654013 LR: 0.0032970502506941557\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training Loss: 0.07037831565918085 Validation loss: 0.638177446343682 LR: 0.003296267008408904\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training Loss: 0.07037018542245073 Validation loss: 0.6374360444870862 LR: 0.0032967643346637487\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training Loss: 0.07036405211309191 Validation loss: 0.6371414376930757 LR: 0.0032959540840238333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training Loss: 0.07033524584959151 Validation loss: 0.637791322036223 LR: 0.003295705420896411\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training Loss: 0.07032263126514143 Validation loss: 0.6372916942292993 LR: 0.00329604628495872\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training Loss: 0.0703654633432369 Validation loss: 0.6376966427672993 LR: 0.003296137321740389\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training Loss: 0.07031634038781226 Validation loss: 0.6379380835728212 LR: 0.0032959922682493925\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training Loss: 0.07034359561005656 Validation loss: 0.6382912614128806 LR: 0.003295617876574397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training Loss: 0.07032723905201947 Validation loss: 0.6377292790196158 LR: 0.0032950658351182938\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training Loss: 0.07032263236018355 Validation loss: 0.6379394368691877 LR: 0.0032948392909020185\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training Loss: 0.07024665621326705 Validation loss: 0.6374831348657608 LR: 0.003294074209406972\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training Loss: 0.07028627461117695 Validation loss: 0.6377922052686865 LR: 0.003294258611276746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training Loss: 0.07027027477920915 Validation loss: 0.6369929300113157 LR: 0.0032942467369139194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training Loss: 0.07029900358458417 Validation loss: 0.6378121010281823 LR: 0.003294066060334444\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training Loss: 0.070252300768833 Validation loss: 0.6387425715273077 LR: 0.003293693531304598\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training Loss: 0.07027322514471128 Validation loss: 0.6369794864546169 LR: 0.0032931820023804903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training Loss: 0.07029680018809756 Validation loss: 0.6387823887846686 LR: 0.0032933801412582397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training Loss: 0.07027706518898093 Validation loss: 0.6381683295423334 LR: 0.003293775487691164\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training Loss: 0.07027030438828881 Validation loss: 0.637236553159627 LR: 0.003292437642812729\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training Loss: 0.07021215575229194 Validation loss: 0.6368065923452377 LR: 0.0032934185583144426\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training Loss: 0.07025433336022265 Validation loss: 0.6375803175297651 LR: 0.003292881418019533\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training Loss: 0.07022980317172811 Validation loss: 0.6374375928531993 LR: 0.0032928218133747578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training Loss: 0.07023640715594938 Validation loss: 0.6386140381748026 LR: 0.003292742883786559\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training Loss: 0.07026664246013253 Validation loss: 0.6375698921355334 LR: 0.0032923452090471983\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training Loss: 0.07021466009211472 Validation loss: 0.6385081329128959 LR: 0.0032920960802584887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training Loss: 0.07020597137317534 Validation loss: 0.6377285732464357 LR: 0.00329204392619431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training Loss: 0.0702096871838446 Validation loss: 0.6372965763915669 LR: 0.0032920099329203367\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training Loss: 0.07020941737405849 Validation loss: 0.6372250088236548 LR: 0.00329182343557477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training Loss: 0.0702138005270051 Validation loss: 0.637881952253255 LR: 0.003291819244623184\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training Loss: 0.07017644347692772 Validation loss: 0.6372643533078107 LR: 0.0032911901362240314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training Loss: 0.07021138905104711 Validation loss: 0.6378042210232128 LR: 0.0032918360084295273\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training Loss: 0.070202094687938 Validation loss: 0.6383135535500266 LR: 0.003291778964921832\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training Loss: 0.07020897744626065 Validation loss: 0.6369289660995657 LR: 0.0032906208653002977\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training Loss: 0.0701921846427732 Validation loss: 0.6373206796971235 LR: 0.0032914939802139997\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training Loss: 0.07019891785269855 Validation loss: 0.637453858147968 LR: 0.0032910045702010393\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training Loss: 0.07021983446666075 Validation loss: 0.6372309530323202 LR: 0.0032907736022025347\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training Loss: 0.0701642646440855 Validation loss: 0.6373927132649855 LR: 0.003290317952632904\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training Loss: 0.07020427142766436 Validation loss: 0.6378022039478476 LR: 0.003290910739451647\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training Loss: 0.0702117459703591 Validation loss: 0.6372696188363162 LR: 0.003289703978225589\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training Loss: 0.07015641217230033 Validation loss: 0.6370982473546808 LR: 0.0032898832578212023\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training Loss: 0.07019852191982764 Validation loss: 0.638265149159865 LR: 0.0032896839547902346\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training Loss: 0.07017487197341424 Validation loss: 0.6380642638965086 LR: 0.0032892082817852497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training Loss: 0.07017784134111761 Validation loss: 0.6378359781070189 LR: 0.003289810148999095\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training Loss: 0.07016027568482529 Validation loss: 0.6374576051126827 LR: 0.00328989140689373\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training Loss: 0.07018482843977573 Validation loss: 0.6375585862181403 LR: 0.003289090935140848\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training Loss: 0.07015585782333822 Validation loss: 0.6371890767054125 LR: 0.0032891978044062853\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training Loss: 0.07019915163345226 Validation loss: 0.6370692903345282 LR: 0.003289175918325782\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training Loss: 0.07017648113778757 Validation loss: 0.6372863745147531 LR: 0.0032886103726923466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training Loss: 0.07017487285374222 Validation loss: 0.6381100199439309 LR: 0.0032887719571590424\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training Loss: 0.07016275209033868 Validation loss: 0.6377128796143965 LR: 0.003288090927526355\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training Loss: 0.0701509304237297 Validation loss: 0.6376407214186408 LR: 0.003288809908553958\n"
     ]
    }
   ],
   "source": [
    "trainLRregression(model_LALR_LC, optimizer_LALR_LC, trainLoader, testLoader, criterion1, tau, N_EPOCHS, lossList_LALR_LC, valList_LALR_LC, \"LC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9590e44e",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training Loss: 3.630483603958438 Validation loss: 1.5025596510280261 LR: 0.12625199556350708\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training Loss: 0.7718808317734116 Validation loss: 0.8411644466898658 LR: 0.031696368008852005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training Loss: 0.7567993366409104 Validation loss: 0.8549009697003798 LR: 0.03169602155685425\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training Loss: 0.7703445768493741 Validation loss: 0.8660090348937295 LR: 0.03170144930481911\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training Loss: 0.7722265505653293 Validation loss: 0.8612823987549002 LR: 0.03170609474182129\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training Loss: 0.76970603160281 Validation loss: 0.8527830906889655 LR: 0.0317130945622921\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training Loss: 0.7648264427349959 Validation loss: 0.8651412183588202 LR: 0.0317191518843174\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training Loss: 0.7681649665667619 Validation loss: 0.8681322972882878 LR: 0.03172392398118973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training Loss: 0.7707946309095157 Validation loss: 0.8680309680375186 LR: 0.031731873750686646\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training Loss: 0.7777718204242695 Validation loss: 0.8647589710625735 LR: 0.0317348837852478\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training Loss: 0.7838627259738163 Validation loss: 0.8768906539136713 LR: 0.03173882141709328\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training Loss: 0.7910070196008819 Validation loss: 0.873808129267259 LR: 0.03173636272549629\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training Loss: 0.801632499145156 Validation loss: 0.8837801665067673 LR: 0.031733188778162\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training Loss: 0.8054116536629646 Validation loss: 0.8840347000143745 LR: 0.031733401119709015\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training Loss: 0.8126711218432665 Validation loss: 0.8940010517835617 LR: 0.03173086792230606\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training Loss: 0.8171817036801868 Validation loss: 0.8850971799005162 LR: 0.03172498568892479\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training Loss: 0.8236196398391504 Validation loss: 0.8846306489272551 LR: 0.03172168508172035\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training Loss: 0.8242508796518749 Validation loss: 0.900293080644174 LR: 0.03171837329864502\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training Loss: 0.8303680177724327 Validation loss: 0.8985968963666395 LR: 0.03171086311340332\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training Loss: 0.833468305446229 Validation loss: 0.899574947628108 LR: 0.03170546516776085\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training Loss: 0.8307239014751973 Validation loss: 0.8962246775627136 LR: 0.03170078992843628\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training Loss: 0.836575305255758 Validation loss: 0.9033411646431143 LR: 0.03169736638665199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training Loss: 0.8317143175031678 Validation loss: 0.8934313316236843 LR: 0.03169507905840874\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training Loss: 0.8345885113611688 Validation loss: 0.8980715843764219 LR: 0.03169170767068863\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training Loss: 0.8351045728416883 Validation loss: 0.9065264869819988 LR: 0.03168719634413719\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training Loss: 0.8378670727485195 Validation loss: 0.9014799635518681 LR: 0.0316857174038887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training Loss: 0.8379530856863566 Validation loss: 0.895991097797047 LR: 0.03168236464262009\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training Loss: 0.8356261141705582 Validation loss: 0.9013142328370701 LR: 0.031681086868047714\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training Loss: 0.8412283775785815 Validation loss: 0.899655977433378 LR: 0.031677089631557465\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training Loss: 0.8365299033500275 Validation loss: 0.8960876383564689 LR: 0.03167792409658432\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training Loss: 0.8379288458343198 Validation loss: 0.9011476906863126 LR: 0.0316765233874321\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training Loss: 0.8396217604879 Validation loss: 0.8947767642411318 LR: 0.03167432174086571\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training Loss: 0.8371986966655303 Validation loss: 0.8972899995066903 LR: 0.03167302533984184\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training Loss: 0.841928172867305 Validation loss: 0.905359839851206 LR: 0.031671445816755295\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training Loss: 0.8379688273245732 Validation loss: 0.8968486528504979 LR: 0.03167019784450531\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training Loss: 0.8394702624862407 Validation loss: 0.8952088952064514 LR: 0.03166801109910011\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training Loss: 0.8413561184055867 Validation loss: 0.900690804828297 LR: 0.031668324023485184\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training Loss: 0.8449775650453155 Validation loss: 0.903720809654756 LR: 0.03166646510362625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training Loss: 0.8388439617857809 Validation loss: 0.910817953673276 LR: 0.03166462108492851\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training Loss: 0.840172248202717 Validation loss: 0.9003053063696081 LR: 0.031665630638599396\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training Loss: 0.8384182880866081 Validation loss: 0.9030258912931789 LR: 0.031665198504924774\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training Loss: 0.8422927055991005 Validation loss: 0.900070308284326 LR: 0.031662989407777786\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training Loss: 0.8379884552886919 Validation loss: 0.90350380404429 LR: 0.03166346624493599\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training Loss: 0.8430997598755257 Validation loss: 0.9007979495958849 LR: 0.031662117689847946\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training Loss: 0.8398497757375756 Validation loss: 0.9109141704711047 LR: 0.03166211023926735\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training Loss: 0.8458447234774873 Validation loss: 0.9050370766357942 LR: 0.03166210651397705\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training Loss: 0.84215831653529 Validation loss: 0.9051099961454218 LR: 0.03166094049811363\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training Loss: 0.8423013419170544 Validation loss: 0.9013954008167441 LR: 0.03166075050830841\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training Loss: 0.8439298012758194 Validation loss: 0.9054552221840079 LR: 0.031659841537475586\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training Loss: 0.840767467915153 Validation loss: 0.9181472550738942 LR: 0.03165948763489723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training Loss: 0.842549192115278 Validation loss: 0.8909611905162985 LR: 0.03165925294160843\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training Loss: 0.8425231364687169 Validation loss: 0.899896277622743 LR: 0.0316590815782547\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training Loss: 0.8453805661338895 Validation loss: 0.9061429243196141 LR: 0.03165785223245621\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training Loss: 0.8468805853846437 Validation loss: 0.89953998002139 LR: 0.03165740147233009\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training Loss: 0.845933604309126 Validation loss: 0.9155361747199838 LR: 0.031657177954912186\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training Loss: 0.8431041829867734 Validation loss: 0.8919379805976694 LR: 0.031656425446271896\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training Loss: 0.8431914649710531 Validation loss: 0.9096868715502999 LR: 0.03165622800588608\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training Loss: 0.8460751919650207 Validation loss: 0.9069993103092368 LR: 0.03165615722537041\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training Loss: 0.8459923610907123 Validation loss: 0.9039913036606528 LR: 0.031655002385377884\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training Loss: 0.8468474773233837 Validation loss: 0.8950903618877585 LR: 0.03165469691157341\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training Loss: 0.8481077235782525 Validation loss: 0.904197795824571 LR: 0.03165488690137863\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training Loss: 0.8476978593669638 Validation loss: 0.9151455949653279 LR: 0.0316549688577652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training Loss: 0.84817403898459 Validation loss: 0.8995547457174822 LR: 0.03165486827492714\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training Loss: 0.8465025002399851 Validation loss: 0.9065296731211923 LR: 0.03165386989712715\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training Loss: 0.8457588029526153 Validation loss: 0.9045559398152612 LR: 0.03165384382009506\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training Loss: 0.8469667340218162 Validation loss: 0.9016722115603361 LR: 0.03165391832590103\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training Loss: 0.8475322860805721 Validation loss: 0.9060403514992107 LR: 0.03165357932448387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training Loss: 0.8491127365604257 Validation loss: 0.900909654118798 LR: 0.031653035432100296\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training Loss: 0.849623063971055 Validation loss: 0.9063875295899131 LR: 0.03165249526500702\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training Loss: 0.8497557655535102 Validation loss: 0.8975354013117877 LR: 0.031652532517910004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training Loss: 0.8457324915385728 Validation loss: 0.8997542709112167 LR: 0.03165178745985031\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training Loss: 0.8485514342956653 Validation loss: 0.9014160497622057 LR: 0.031652264297008514\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training Loss: 0.8493062687195 Validation loss: 0.9159381823106245 LR: 0.03165186941623688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training Loss: 0.8475858764964497 Validation loss: 0.9065937345678156 LR: 0.031652066856622696\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training Loss: 0.8483451917466925 Validation loss: 0.9065125042741949 LR: 0.03165088966488838\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training Loss: 0.8520131667684065 Validation loss: 0.9058828787370161 LR: 0.03165125101804733\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training Loss: 0.8456819871660611 Validation loss: 0.9070937660607424 LR: 0.03165103495121002\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training Loss: 0.8498647666458438 Validation loss: 0.9020969298752871 LR: 0.03165114298462868\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training Loss: 0.8487838523875739 Validation loss: 0.9089781194925308 LR: 0.03165002539753914\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training Loss: 0.8515993238182508 Validation loss: 0.9033365005796606 LR: 0.03165087848901749\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training Loss: 0.8499882702181594 Validation loss: 0.893673453818668 LR: 0.03165046125650406\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training Loss: 0.850486758291206 Validation loss: 0.9154007868333296 LR: 0.03164919838309288\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training Loss: 0.8508303278804856 Validation loss: 0.9051428518512032 LR: 0.03164917975664139\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training Loss: 0.8508396653689294 Validation loss: 0.9104129685596987 LR: 0.031649149954319\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training Loss: 0.8525556676325949 Validation loss: 0.9052833698012612 LR: 0.03164956718683243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training Loss: 0.8508376221491899 Validation loss: 0.9032257361845537 LR: 0.031649135053157806\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training Loss: 0.8504357224582595 Validation loss: 0.8998616568066857 LR: 0.03164861351251602\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training Loss: 0.8536882628968538 Validation loss: 0.9040377993475307 LR: 0.031649406999349594\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training Loss: 0.8490384320704325 Validation loss: 0.9118737524205988 LR: 0.03164830058813095\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training Loss: 0.8502472855515714 Validation loss: 0.9031201397830789 LR: 0.0316489078104496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training Loss: 0.8493723905395706 Validation loss: 0.8997432169589129 LR: 0.031648751348257065\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training Loss: 0.8489825569243528 Validation loss: 0.899257632819089 LR: 0.03164787217974663\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training Loss: 0.8539855583256878 Validation loss: 0.898393989963965 LR: 0.03164803609251976\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training Loss: 0.8515205192634626 Validation loss: 0.9052081392569975 LR: 0.03164765238761902\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training Loss: 0.853337253033256 Validation loss: 0.908753521063111 LR: 0.03164870664477348\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training Loss: 0.8511959283427477 Validation loss: 0.8991540724580939 LR: 0.03164741396903992\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training Loss: 0.8501954119899431 Validation loss: 0.9103845384987918 LR: 0.03164736554026604\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training Loss: 0.8514633316127986 Validation loss: 0.9038113965229555 LR: 0.03164738416671753\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training Loss: 0.8522159759524233 Validation loss: 0.9104465625502847 LR: 0.031647417694330215\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training Loss: 0.8524367714821434 Validation loss: 0.9071399840441617 LR: 0.03164784982800484\n"
     ]
    }
   ],
   "source": [
    "trainLRregression(model_LALR_L1, optimizer_LALR_L1, trainLoader, testLoader, criterion2, tau, N_EPOCHS, lossList_LALR_L1, valList_LALR_L1, \"L1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "76948d0c",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 Training Loss: 0.40664654841340586 Validation loss: 0.6346689625219866 LR: 0.0006591904675588012\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Training Loss: 0.3344127631153085 Validation loss: 0.6251909312876788 LR: 0.00018401831039227545\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 Training Loss: 0.3270726753586651 Validation loss: 0.6217242912812666 LR: 0.00017724515055306256\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 Training Loss: 0.3226577828528215 Validation loss: 0.619739215482365 LR: 0.00019324672757647932\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 Training Loss: 0.31874736404899906 Validation loss: 0.6175122708082199 LR: 0.00017757731257006526\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 Training Loss: 0.3162547105155692 Validation loss: 0.6160810833627527 LR: 0.00018680634093470871\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 Training Loss: 0.31399773924426316 Validation loss: 0.6151632382111116 LR: 0.0001873129658633843\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 Training Loss: 0.31199905963383767 Validation loss: 0.6139193841002204 LR: 0.0001796537690097466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 Training Loss: 0.3103874309433984 Validation loss: 0.6126782921227542 LR: 0.00017762131756171584\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 Training Loss: 0.3090561403321258 Validation loss: 0.6117132089354775 LR: 0.00018355291103944182\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Training Loss: 0.3079574594236588 Validation loss: 0.6113691221583973 LR: 0.000191026134416461\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Training Loss: 0.30663409936496777 Validation loss: 0.6103307862173427 LR: 0.00018492917297407985\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Training Loss: 0.30541628650698266 Validation loss: 0.6091087446971373 LR: 0.0001776802382664755\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Training Loss: 0.3044281194807817 Validation loss: 0.6086524332111533 LR: 0.0001740560110192746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Training Loss: 0.3035775788922818 Validation loss: 0.6079422492872585 LR: 0.00017522514099255204\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Training Loss: 0.302954967238374 Validation loss: 0.6074498309330507 LR: 0.00018230907153338194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Training Loss: 0.3021121945779674 Validation loss: 0.6066825728524815 LR: 0.00018256381736136973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Training Loss: 0.30124004945287786 Validation loss: 0.6064356443556872 LR: 0.00017705919162835926\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Training Loss: 0.3008296658480889 Validation loss: 0.6061213748021559 LR: 0.0001900933712022379\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Training Loss: 0.299892627196628 Validation loss: 0.6053804552013223 LR: 0.00017767789540812373\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Training Loss: 0.2991306236394888 Validation loss: 0.6046691577542912 LR: 0.00017147294420283288\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Training Loss: 0.298677328555316 Validation loss: 0.6045194877819582 LR: 0.00017730142280925065\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Training Loss: 0.29822772964620453 Validation loss: 0.6042232472788204 LR: 0.00018549847300164402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Training Loss: 0.29759600792906815 Validation loss: 0.603629014708779 LR: 0.00017853654571808875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Training Loss: 0.29677474412863125 Validation loss: 0.602991449561986 LR: 0.00016790955851320177\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Training Loss: 0.29609233334188156 Validation loss: 0.6026152711022984 LR: 0.00015821534907445312\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Training Loss: 0.29589127583874747 Validation loss: 0.6024984243241224 LR: 0.0001707542542135343\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Training Loss: 0.29545339486784483 Validation loss: 0.6021882837468927 LR: 0.00017020656378008425\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Training Loss: 0.2951214119405499 Validation loss: 0.602051863616163 LR: 0.00017365410167258233\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Training Loss: 0.2945230352431965 Validation loss: 0.6018004579977556 LR: 0.00016443054482806474\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Training Loss: 0.2941886104578931 Validation loss: 0.6014566665345972 LR: 0.00016710573981981725\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Training Loss: 0.29384238730246465 Validation loss: 0.6013118096373298 LR: 0.00016579929797444493\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Training Loss: 0.29341134235735244 Validation loss: 0.6007942178032615 LR: 0.00016369765216950327\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Training Loss: 0.29298600009092335 Validation loss: 0.6003783020106229 LR: 0.00016007373051252216\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Training Loss: 0.29268642018954416 Validation loss: 0.5998640250075947 LR: 0.00016158576181624085\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Training Loss: 0.29244619814050987 Validation loss: 0.6000714681365273 LR: 0.00016531063010916114\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Training Loss: 0.2920518307074346 Validation loss: 0.5995789983055808 LR: 0.00016091509314719588\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Training Loss: 0.29165981405063046 Validation loss: 0.5994076092134822 LR: 0.00015718452050350606\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Training Loss: 0.29145785938594115 Validation loss: 0.5993664264678955 LR: 0.00016373353719245642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Training Loss: 0.2912155769400363 Validation loss: 0.5990645397793163 LR: 0.00016018639144022018\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 40 Training Loss: 0.29092921565692087 Validation loss: 0.5991943424398248 LR: 0.0001614174252608791\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 41 Training Loss: 0.29060680786367793 Validation loss: 0.5984470993280411 LR: 0.00015846277528908104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 42 Training Loss: 0.29058457039275154 Validation loss: 0.5987129387530413 LR: 0.00016761349979788065\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 43 Training Loss: 0.29023844295688594 Validation loss: 0.5985587415370074 LR: 0.00016510333807673305\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 44 Training Loss: 0.2898551994768275 Validation loss: 0.5979717279022391 LR: 0.0001597463560756296\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 45 Training Loss: 0.2898390994807142 Validation loss: 0.5981221009384502 LR: 0.00016699494153726846\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 46 Training Loss: 0.2894067351601653 Validation loss: 0.5979174930940975 LR: 0.00015874157543294132\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 47 Training Loss: 0.2893708795738495 Validation loss: 0.5976504182273691 LR: 0.0001666969619691372\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 48 Training Loss: 0.28894300230641873 Validation loss: 0.5976683659987017 LR: 0.0001578472729306668\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 49 Training Loss: 0.28876362710933523 Validation loss: 0.5974927341396158 LR: 0.00016192844486795366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 50 Training Loss: 0.28859179516347067 Validation loss: 0.5976758016781374 LR: 0.0001608375459909439\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 51 Training Loss: 0.28815858667281585 Validation loss: 0.5971476462754336 LR: 0.00015184401127044111\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 52 Training Loss: 0.287951938271179 Validation loss: 0.5965751964937557 LR: 0.0001516249612905085\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 53 Training Loss: 0.28788428145973416 Validation loss: 0.5970493866638704 LR: 0.00015740511298645288\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 54 Training Loss: 0.28798098905114 Validation loss: 0.5967057550495322 LR: 0.00016816802963148803\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 55 Training Loss: 0.2875220370223955 Validation loss: 0.5960841151801023 LR: 0.00015587909729219973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 56 Training Loss: 0.2871841918838127 Validation loss: 0.5967146307229996 LR: 0.00015064251783769578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 57 Training Loss: 0.28715940786026395 Validation loss: 0.5960668826645071 LR: 0.00015841780987102538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 58 Training Loss: 0.28688850213024736 Validation loss: 0.5957808508114382 LR: 0.00015198730397969484\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 59 Training Loss: 0.28686331951137234 Validation loss: 0.5963745076547969 LR: 0.000160219642566517\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 60 Training Loss: 0.2866280135143044 Validation loss: 0.595714970068498 LR: 0.00015613585128448904\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 61 Training Loss: 0.28655793915049144 Validation loss: 0.5956146405501799 LR: 0.0001590611645951867\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 62 Training Loss: 0.28627991100896677 Validation loss: 0.595710738138719 LR: 0.0001543312391731888\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 63 Training Loss: 0.28624896657913496 Validation loss: 0.5958960394967686 LR: 0.00015897316916380078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 64 Training Loss: 0.286033409816731 Validation loss: 0.595355895432559 LR: 0.0001573529007146135\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 65 Training Loss: 0.28593945799540377 Validation loss: 0.5952120030468161 LR: 0.00016001462063286453\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 66 Training Loss: 0.28577974670558565 Validation loss: 0.595318307930773 LR: 0.0001581350079504773\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 67 Training Loss: 0.28559752152185963 Validation loss: 0.5951195494695143 LR: 0.00015578947204630822\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 68 Training Loss: 0.285316726230407 Validation loss: 0.5949905921112407 LR: 0.0001525359693914652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 69 Training Loss: 0.2853207788738806 Validation loss: 0.594916893677278 LR: 0.00015796541993040591\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 70 Training Loss: 0.285074209359606 Validation loss: 0.5950981256636706 LR: 0.00015433054068125784\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 71 Training Loss: 0.2850096412969254 Validation loss: 0.5950798216191205 LR: 0.00015583100321236998\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72 Training Loss: 0.28495909934943964 Validation loss: 0.5952035893093456 LR: 0.00015717001224402338\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 73 Training Loss: 0.28480224118109054 Validation loss: 0.5944706987250935 LR: 0.00015726740821264684\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 74 Training Loss: 0.2846126870737983 Validation loss: 0.5949106175791133 LR: 0.00015349312161561102\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 75 Training Loss: 0.28463974857021135 Validation loss: 0.594639778137207 LR: 0.00015819718828424811\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 76 Training Loss: 0.2844333105025443 Validation loss: 0.5944656038826163 LR: 0.0001550727611174807\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 77 Training Loss: 0.284362877437292 Validation loss: 0.5941597114909779 LR: 0.00015702040400356054\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 78 Training Loss: 0.2842045988490327 Validation loss: 0.59416034004905 LR: 0.00015306903515011072\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 79 Training Loss: 0.28412934008352353 Validation loss: 0.594227369536053 LR: 0.00015654366870876402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 80 Training Loss: 0.2838996401437765 Validation loss: 0.5937190516428514 LR: 0.00014924553397577256\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 81 Training Loss: 0.2837667946169631 Validation loss: 0.5939893641255118 LR: 0.0001484462700318545\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 82 Training Loss: 0.28380045260407394 Validation loss: 0.5941976322369142 LR: 0.00015550652460660785\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 83 Training Loss: 0.2837169359757509 Validation loss: 0.5939207971096039 LR: 0.00015290694136638194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 84 Training Loss: 0.283581102658761 Validation loss: 0.5942106138576161 LR: 0.0001541249075671658\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 85 Training Loss: 0.2835205207640568 Validation loss: 0.5937998701225627 LR: 0.00015455343236681074\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 86 Training Loss: 0.2834547542529422 Validation loss: 0.593791888518767 LR: 0.0001544299302622676\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 87 Training Loss: 0.2834753517372807 Validation loss: 0.593988994305784 LR: 0.00015818836982361972\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 88 Training Loss: 0.2832546733985373 Validation loss: 0.5937523259357973 LR: 0.0001541756500955671\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 89 Training Loss: 0.28330935975831933 Validation loss: 0.5936435149474577 LR: 0.00015869797789491713\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 90 Training Loss: 0.28311002198660407 Validation loss: 0.5934649055654352 LR: 0.00015368065214715898\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 91 Training Loss: 0.28321222248098005 Validation loss: 0.5935685119845651 LR: 0.00016270269406959414\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 92 Training Loss: 0.28298398427214333 Validation loss: 0.5934663293036547 LR: 0.00015381033881567419\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 93 Training Loss: 0.28292173237381474 Validation loss: 0.5932003544135527 LR: 0.00015686792903579772\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 94 Training Loss: 0.28288003643411036 Validation loss: 0.593355342745781 LR: 0.0001549459993839264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 95 Training Loss: 0.2827273262423122 Validation loss: 0.5935922305692326 LR: 0.0001545351551612839\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 96 Training Loss: 0.28262639775743403 Validation loss: 0.5931435227394104 LR: 0.00015264676767401397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 97 Training Loss: 0.2825737674143541 Validation loss: 0.5933584015477787 LR: 0.00015424928278662264\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 98 Training Loss: 0.28252685791477344 Validation loss: 0.5934527326713909 LR: 0.00015358190285041928\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 99 Training Loss: 0.28243514036582595 Validation loss: 0.5932127277959477 LR: 0.00015261504449881613\n"
     ]
    }
   ],
   "source": [
    "trainLRregression(model_LALR_MSE, optimizer_LALR_MSE, trainLoader, testLoader, criterion3, tau, N_EPOCHS, lossList_LALR_MSE, valList_LALR_MSE, \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45ee6a44",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# LBFGS training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fca9946",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trainLBFGS(model_LBFGS_LC, optimizer_LBFGS_LC, trainLoader, testLoader, criterion1, tau, N_EPOCHS, lossList_LBFGS_LC, valList_LBFGS_LC, \"LC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d92a868",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trainLBFGS(model_LBFGS_MSE, optimizer_LBFGS_MSE, trainLoader, testLoader, criterion3, tau, N_EPOCHS, lossList_LBFGS_MSE, valList_LBFGS_MSE, \"MSE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3f5d1f0",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Saving losses and models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c18f7696",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A= np.asarray(lossList_CLR_LC)\n",
    "B= np.asarray(lossList_LALR_LC)\n",
    "# C= np.asarray(lossList_LBFGS_LC)\n",
    "\n",
    "A_= np.asarray(lossList_CLR_L1)\n",
    "B_= np.asarray(lossList_LALR_L1)\n",
    "# C_= np.asarray(lossList_LBFGS_L1)\n",
    "\n",
    "A__= np.asarray(lossList_CLR_MSE)\n",
    "B__= np.asarray(lossList_LALR_MSE)\n",
    "# C__= np.asarray(lossList_LBFGS_MSE)\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_CLR_LC.npy\", A)\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_LALR_LC.npy\", B)\n",
    "# np.save(\"./D-GEX_checkpoint/LossLists/lossList_LBFGS_LC.npy\", C)\n",
    "\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_CLR_L1.npy\", A_)\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_LALR_L1.npy\", B_)\n",
    "# np.save(\"./D-GEX_checkpoint/LossLists/lossList_LBFGS_L1.npy\", C_)\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_CLR_MSE.npy\", A__)\n",
    "np.save(\"./D-GEX_checkpoint/LossLists/lossList_LALR_MSE.npy\", B__)\n",
    "# np.save(\"./D-GEX_checkpoint/LossLists/lossList_LBFGS_MSE.npy\", C__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "26812480",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "A= np.asarray(valList_CLR_LC)\n",
    "B= np.asarray(valList_LALR_LC)\n",
    "# C= np.asarray(valList_LBFGS_LC)\n",
    "\n",
    "A_= np.asarray(valList_CLR_L1)\n",
    "B_= np.asarray(valList_LALR_L1)\n",
    "# C_= np.asarray(valList_LBFGS_L1)\n",
    "\n",
    "A__= np.asarray(valList_CLR_MSE)\n",
    "B__= np.asarray(valList_LALR_MSE)\n",
    "# C__= np.asarray(valList_LBFGS_MSE)\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_CLR_LC.npy\", A)\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_LALR_LC.npy\", B)\n",
    "# np.save(\"./D-GEX_checkpoint/ValLists/valList_LBFGS_LC.npy\", C)\n",
    "\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_CLR_L1.npy\", A_)\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_LALR_L1.npy\", B_)\n",
    "# np.save(\"./D-GEX_checkpoint/ValLists/valList_LBFGS_L1.npy\", C_)\n",
    "\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_CLR_MSE.npy\", A__)\n",
    "np.save(\"./D-GEX_checkpoint/ValLists/valList_LALR_MSE.npy\", B__)\n",
    "# np.save(\"./D-GEX_checkpoint/ValLists/valList_LBFGS_MSE.npy\", C__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "baf09c67",
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "torch.save(model_CLR_LC.state_dict(), \"./D-GEX_checkpoint/model_params/CLR_LC.pt\")\n",
    "torch.save(model_LALR_LC.state_dict(), \"./D-GEX_checkpoint/model_params/LALR_LC.pt\")\n",
    "# torch.save(model_LBFGS_LC.state_dict(), \"./D-GEX_checkpoint/model_params/LBFGS_LC.pt\")\n",
    "\n",
    "torch.save(model_CLR_L1.state_dict(), \"./D-GEX_checkpoint/model_params/CLR_L1.pt\")\n",
    "torch.save(model_LALR_L1.state_dict(), \"./D-GEX_checkpoint/model_params/LALR_L1.pt\")\n",
    "# torch.save(model_LBFGS_L1.state_dict(), \"./D-GEX_checkpoint/model_params/LBFGS_L1.pt\")\n",
    "\n",
    "torch.save(model_CLR_MSE.state_dict(), \"./D-GEX_checkpoint/model_params/CLR_MSE.pt\")\n",
    "torch.save(model_LALR_MSE.state_dict(), \"./D-GEX_checkpoint/model_params/LALR_MSE.pt\")\n",
    "# torch.save(model_LBFGS_MSE.state_dict(), \"./D-GEX_checkpoint/model_params/LBFGS_MSE.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9120c2fd",
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0 0 1 0 0\n"
     ]
    }
   ],
   "source": [
    "# taking min RMSE:\n",
    "minRMSE= min(min(valList_CLR_LC), min(valList_LALR_LC), min(valList_CLR_L1), min(valList_LALR_L1), min(valList_CLR_MSE), min(valList_LALR_MSE))\n",
    "epoch_CLR_LC, epoch_LALR_LC, epoch_CLR_L1, epoch_LALR_L1, epoch_CLR_MSE, epoch_LALR_MSE= [], [], [], [], [], []\n",
    "for idx in range(100):\n",
    "    epoch_CLR_LC.append(minRMSE/valList_CLR_LC[idx]*100)\n",
    "    epoch_LALR_LC.append(minRMSE/valList_LALR_LC[idx]*100)\n",
    "    epoch_CLR_L1.append(minRMSE/valList_CLR_L1[idx]*100)\n",
    "    epoch_LALR_L1.append(minRMSE/valList_LALR_L1[idx]*100)\n",
    "    epoch_CLR_MSE.append(minRMSE/valList_CLR_MSE[idx]*100)\n",
    "    epoch_LALR_MSE.append(minRMSE/valList_LALR_MSE[idx]*100)\n",
    "\n",
    "# idx= epoch_CLR_LC.index(max(epoch_CLR_LC))\n",
    "minMaxRMSE= min(max(epoch_CLR_LC), max(epoch_LALR_LC), max(epoch_CLR_L1), max(epoch_LALR_L1), max(epoch_CLR_MSE), max(epoch_LALR_MSE))\n",
    "\n",
    "# calculating number of epochs taken to reach minMaxRMSE for each\n",
    "def qH(lst, minMax):\n",
    "    for idx, item in enumerate(lst):\n",
    "        if lst[idx]>= minMax:\n",
    "            return idx\n",
    "CLR_LC, LALR_LC, CLR_L1, LALR_L1, CLR_MSE, LALR_MSE= qH(epoch_CLR_LC, minMaxRMSE), qH(epoch_LALR_LC, minMaxRMSE), qH(epoch_CLR_L1, minMaxRMSE), qH(epoch_LALR_L1,minMaxRMSE), qH(epoch_CLR_MSE,minMaxRMSE), qH(epoch_LALR_MSE,minMaxRMSE)\n",
    "print(CLR_LC, LALR_LC, CLR_L1, LALR_L1, CLR_MSE, LALR_MSE)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "argv": [
    "/home/aryamanj/miniconda3/bin/python",
    "-m",
    "ipykernel_launcher",
    "-f",
    "{connection_file}"
   ],
   "display_name": "Python 3 (ipykernel)",
   "env": null,
   "interrupt_mode": "signal",
   "language": "python",
   "metadata": {
    "debugger": true
   },
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "name": "GEOLoaderFinal.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 5
}