cifar.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from collections import OrderedDict, defaultdict\n",
    "import torch.optim as optim\n",
    "import time\n",
    "from src import *\n",
    "import math\n",
    "import pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rc('text', usetex=True)\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "!mkdir -p figures\n",
    "!mkdir -p snapshots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "# PGK\n",
    "ϵ = 8 / 256\n",
    "ϵ_s = 2 / 256\n",
    "\n",
    "\n",
    "val_K = 10\n",
    "EPOCHS = 200\n",
    "TEST_EVERY = 40\n",
    "\n",
    "batch_size = 128\n",
    "pre_train = False\n",
    "\n",
    "small = False\n",
    "training_with_replay_Ks = [1, 4, 10, 20]\n",
    "free_Ks = [1, 2, 4, 10]\n",
    "\n",
    "    \n",
    "PGD_Ks = [1, 2]\n",
    "\n",
    "\n",
    "\n",
    "attack_names = ['FSM', 'PGD-20', 'PGD-100', 'CW-100']\n",
    "attacks = [\n",
    "     *[PGD(K, ϵ, 2.5 * ϵ/K) for K in [1, 20, 100]],\n",
    "     CW(100, 1e4, ϵ, 2.5 * ϵ/ 100)]\n",
    "    \n",
    "    \n",
    "if small:\n",
    "    EPOCHS = 5\n",
    "    TEST_EVERY = 5\n",
    "    training_with_replay_Ks = [1, 5]\n",
    "    free_Ks = [1, 5]\n",
    "    attack_names = ['FSM', 'PGD-2', 'CW-2']\n",
    "    attacks = [\n",
    "         *[PGD(K, ϵ, 2.5 * ϵ/K) for K in [1, 2]],\n",
    "         CW(2, 1e4, ϵ, 2.5 * ϵ/ 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(EPOCHS == K * int(EPOCHS / K) for K in training_with_replay_Ks)\n",
    "assert all(EPOCHS == K * int(EPOCHS / K) for K in free_Ks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# CIFAR INPUT\n",
    "transform = transforms.Compose(\n",
    "    [transforms.RandomCrop(32, padding=4),\n",
    "     transforms.ToTensor(),\n",
    "     transforms.RandomHorizontalFlip(p=0.5),\n",
    "    ])\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True, \n",
    "                                        download=True, transform=transform)\n",
    "if small:\n",
    "    trainset = torch.utils.data.Subset(trainset, range(batch_size))\n",
    "\n",
    "trainloader = torch.utils.data.DataLoader(trainset, \n",
    "        batch_size=batch_size,\n",
    "        shuffle=True, num_workers=4, \n",
    "        pin_memory=True, drop_last=True)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform_test)\n",
    "\n",
    "if small:\n",
    "    testset = torch.utils.data.Subset(testset, range(batch_size))\n",
    "    \n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=4)\n",
    "\n",
    "\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm = StandardScalerLayer(lambda: map(lambda x: x[0], trainloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 32, 32])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def build_model(ϵ=ϵ, K=1):\n",
    "    model = WideResNet(28, 10, 10, 0.1)\n",
    "    adv = AdversarialForFree(ϵ, 0, 1)\n",
    "    if ϵ not in [0, False]:\n",
    "        l = [('adv', adv)]\n",
    "    else:\n",
    "        l = []\n",
    "    l.extend([\n",
    "        ('normalizer', norm),\n",
    "        ('resnet', model)])\n",
    "    \n",
    "    model = nn.Sequential(OrderedDict(l)).cuda()\n",
    "    \n",
    "    optimizer = optim.SGD(model.parameters(), \n",
    "                          lr=0.1,\n",
    "                          nesterov=True, \n",
    "                          momentum=0.9)\n",
    "    \n",
    "    scheduler =  optim.lr_scheduler.MultiStepLR(optimizer, milestones=[60//K, 120//K, 160//K], gamma=0.2)\n",
    "    \n",
    "    return model, optimizer, scheduler\n",
    "\n",
    "imgsize = images.size()[1:]\n",
    "imgsize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 1 replays------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 1.4239 48.3% 70.5s\n",
      "train \t 2: 0.9001 68.2% 70.4s\n",
      "train \t 3: 0.7045 75.5% 70.5s\n",
      "train \t 4: 0.5947 79.3% 70.5s\n",
      "train \t 5: 0.5242 81.7% 70.5s\n",
      "train \t 6: 0.4688 83.9% 71.0s\n",
      "train \t 7: 0.4235 85.3% 70.5s\n",
      "train \t 8: 0.3927 86.3% 70.5s\n",
      "train \t 9: 0.3571 87.6% 70.5s\n",
      "train \t 10: 0.3299 88.6% 70.5s\n",
      "train \t 11: 0.3091 89.2% 70.5s\n",
      "train \t 12: 0.2870 89.9% 70.5s\n",
      "train \t 13: 0.2657 90.8% 70.6s\n",
      "train \t 14: 0.2475 91.4% 70.9s\n",
      "train \t 15: 0.2348 91.8% 70.5s\n",
      "train \t 16: 0.2170 92.3% 70.5s\n",
      "train \t 17: 0.2089 92.7% 70.9s\n",
      "train \t 18: 0.1956 93.2% 70.5s\n",
      "train \t 19: 0.1847 93.6% 70.5s\n",
      "train \t 20: 0.1736 93.8% 70.5s\n",
      "train \t 21: 0.1627 94.2% 70.5s\n",
      "train \t 22: 0.1493 94.7% 70.5s\n",
      "train \t 23: 0.1438 94.9% 70.5s\n",
      "train \t 24: 0.1347 95.2% 70.5s\n",
      "train \t 25: 0.1252 95.5% 70.5s\n",
      "train \t 26: 0.1222 95.7% 70.5s\n",
      "train \t 27: 0.1158 95.9% 70.5s\n",
      "train \t 28: 0.1060 96.2% 70.5s\n",
      "train \t 29: 0.1054 96.3% 70.5s\n",
      "train \t 30: 0.0952 96.6% 70.5s\n",
      "train \t 31: 0.0900 96.8% 70.5s\n",
      "train \t 32: 0.0903 96.7% 70.5s\n",
      "train \t 33: 0.0831 97.1% 70.5s\n",
      "train \t 34: 0.0788 97.2% 70.9s\n",
      "train \t 35: 0.0738 97.4% 70.5s\n",
      "train \t 36: 0.0701 97.5% 70.9s\n",
      "train \t 37: 0.0694 97.5% 70.5s\n",
      "train \t 38: 0.0697 97.6% 70.5s\n",
      "train \t 39: 0.0634 97.8% 70.5s\n",
      "train \t 40: 0.0600 97.8% 70.9s\n",
      "val \t 40: 0.4198 90.4% 4.9s\n",
      "train \t 41: 0.0579 98.0% 70.5s\n",
      "train \t 42: 0.0519 98.1% 70.5s\n",
      "train \t 43: 0.0508 98.2% 70.9s\n",
      "train \t 44: 0.0469 98.3% 70.4s\n",
      "train \t 45: 0.0510 98.2% 70.5s\n",
      "train \t 46: 0.0484 98.3% 70.5s\n",
      "train \t 47: 0.0438 98.5% 70.9s\n",
      "train \t 48: 0.0399 98.6% 70.5s\n",
      "train \t 49: 0.0418 98.6% 70.5s\n",
      "train \t 50: 0.0426 98.5% 70.5s\n",
      "train \t 51: 0.0351 98.7% 70.5s\n",
      "train \t 52: 0.0379 98.7% 70.5s\n",
      "train \t 53: 0.0376 98.6% 70.5s\n",
      "train \t 54: 0.0374 98.7% 70.5s\n",
      "train \t 55: 0.0336 98.8% 70.6s\n",
      "train \t 56: 0.0348 98.8% 70.5s\n",
      "train \t 57: 0.0322 98.9% 70.5s\n",
      "train \t 58: 0.0273 99.0% 70.5s\n",
      "train \t 59: 0.0270 99.1% 70.5s\n",
      "train \t 60: 0.0292 99.0% 70.5s\n",
      "train \t 61: 0.0148 99.5% 70.5s\n",
      "train \t 62: 0.0080 99.8% 70.5s\n",
      "train \t 63: 0.0067 99.8% 70.5s\n",
      "train \t 64: 0.0067 99.8% 70.9s\n",
      "train \t 65: 0.0057 99.8% 70.9s\n",
      "train \t 66: 0.0050 99.9% 70.8s\n",
      "train \t 67: 0.0049 99.9% 70.9s\n",
      "train \t 68: 0.0046 99.9% 70.9s\n",
      "train \t 69: 0.0044 99.9% 70.4s\n",
      "train \t 70: 0.0046 99.9% 70.9s\n",
      "train \t 71: 0.0041 99.9% 70.5s\n",
      "train \t 72: 0.0039 99.9% 70.5s\n",
      "train \t 73: 0.0043 99.9% 70.5s\n",
      "train \t 74: 0.0029 99.9% 70.5s\n",
      "train \t 75: 0.0034 99.9% 70.5s\n",
      "train \t 76: 0.0033 99.9% 70.5s\n",
      "train \t 77: 0.0027 99.9% 70.5s\n",
      "train \t 78: 0.0030 99.9% 70.5s\n",
      "train \t 79: 0.0031 99.9% 70.5s\n",
      "train \t 80: 0.0032 99.9% 70.5s\n",
      "val \t 80: 0.3804 93.4% 4.9s\n",
      "train \t 81: 0.0027 99.9% 70.4s\n",
      "train \t 82: 0.0027 99.9% 70.4s\n",
      "train \t 83: 0.0025 99.9% 70.5s\n",
      "train \t 84: 0.0025 99.9% 70.5s\n",
      "train \t 85: 0.0030 99.9% 70.5s\n",
      "train \t 86: 0.0028 99.9% 70.5s\n",
      "train \t 87: 0.0023 99.9% 70.5s\n",
      "train \t 88: 0.0023 99.9% 70.5s\n",
      "train \t 89: 0.0024 99.9% 70.5s\n",
      "train \t 90: 0.0021 100.0% 70.5s\n",
      "train \t 91: 0.0031 99.9% 70.5s\n",
      "train \t 92: 0.0026 99.9% 70.5s\n",
      "train \t 93: 0.0022 99.9% 70.5s\n",
      "train \t 94: 0.0024 99.9% 70.5s\n",
      "train \t 95: 0.0022 99.9% 70.5s\n",
      "train \t 96: 0.0017 100.0% 70.5s\n",
      "train \t 97: 0.0023 99.9% 70.9s\n",
      "train \t 98: 0.0015 100.0% 70.5s\n",
      "train \t 99: 0.0019 100.0% 70.5s\n",
      "train \t 100: 0.0014 100.0% 70.9s\n",
      "train \t 101: 0.0018 100.0% 70.5s\n",
      "train \t 102: 0.0018 100.0% 70.9s\n",
      "train \t 103: 0.0017 100.0% 70.4s\n",
      "train \t 104: 0.0018 100.0% 70.5s\n",
      "train \t 105: 0.0017 100.0% 70.5s\n",
      "train \t 106: 0.0021 99.9% 70.5s\n",
      "train \t 107: 0.0018 99.9% 70.9s\n",
      "train \t 108: 0.0016 100.0% 70.5s\n",
      "train \t 109: 0.0021 99.9% 70.5s\n",
      "train \t 110: 0.0019 99.9% 70.9s\n",
      "train \t 111: 0.0018 100.0% 70.9s\n",
      "train \t 112: 0.0022 99.9% 70.9s\n",
      "train \t 113: 0.0016 100.0% 70.5s\n",
      "train \t 114: 0.0017 100.0% 70.5s\n",
      "train \t 115: 0.0013 100.0% 70.5s\n",
      "train \t 116: 0.0018 100.0% 70.6s\n",
      "train \t 117: 0.0016 100.0% 70.5s\n",
      "train \t 118: 0.0013 100.0% 70.9s\n",
      "train \t 119: 0.0015 100.0% 70.9s\n",
      "train \t 120: 0.0016 100.0% 70.9s\n",
      "val \t 120: 0.4152 93.6% 4.9s\n",
      "train \t 121: 0.0017 99.9% 70.5s\n",
      "train \t 122: 0.0013 100.0% 70.9s\n",
      "train \t 123: 0.0013 100.0% 70.9s\n",
      "train \t 124: 0.0013 100.0% 70.5s\n",
      "train \t 125: 0.0014 100.0% 70.5s\n",
      "train \t 126: 0.0016 100.0% 70.8s\n",
      "train \t 127: 0.0012 100.0% 70.5s\n",
      "train \t 128: 0.0012 100.0% 70.5s\n",
      "train \t 129: 0.0012 100.0% 70.5s\n",
      "train \t 130: 0.0017 99.9% 70.5s\n",
      "train \t 131: 0.0014 100.0% 70.5s\n",
      "train \t 132: 0.0015 100.0% 70.5s\n",
      "train \t 133: 0.0011 100.0% 70.9s\n",
      "train \t 134: 0.0013 100.0% 70.5s\n",
      "train \t 135: 0.0011 100.0% 70.5s\n",
      "train \t 136: 0.0008 100.0% 70.5s\n",
      "train \t 137: 0.0015 100.0% 70.5s\n",
      "train \t 138: 0.0011 100.0% 70.9s\n",
      "train \t 139: 0.0009 100.0% 70.5s\n",
      "train \t 140: 0.0011 100.0% 70.5s\n",
      "train \t 141: 0.0010 100.0% 70.9s\n",
      "train \t 142: 0.0013 100.0% 70.9s\n",
      "train \t 143: 0.0011 100.0% 70.9s\n",
      "train \t 144: 0.0011 100.0% 70.6s\n",
      "train \t 145: 0.0010 100.0% 70.5s\n",
      "train \t 146: 0.0011 100.0% 70.9s\n",
      "train \t 147: 0.0015 100.0% 70.5s\n",
      "train \t 148: 0.0011 100.0% 70.9s\n",
      "train \t 149: 0.0013 100.0% 70.5s\n",
      "train \t 150: 0.0011 100.0% 70.5s\n",
      "train \t 151: 0.0014 100.0% 70.9s\n",
      "train \t 152: 0.0008 100.0% 70.5s\n",
      "train \t 153: 0.0012 100.0% 70.5s\n",
      "train \t 154: 0.0010 100.0% 70.5s\n",
      "train \t 155: 0.0010 100.0% 70.5s\n",
      "train \t 156: 0.0008 100.0% 70.9s\n",
      "train \t 157: 0.0012 100.0% 70.6s\n",
      "train \t 158: 0.0012 100.0% 70.5s\n",
      "train \t 159: 0.0011 100.0% 70.5s\n",
      "train \t 160: 0.0011 100.0% 70.5s\n",
      "val \t 160: 0.4077 93.7% 4.9s\n",
      "train \t 161: 0.0012 100.0% 70.5s\n",
      "train \t 162: 0.0012 100.0% 70.5s\n",
      "train \t 163: 0.0011 100.0% 70.5s\n",
      "train \t 164: 0.0009 100.0% 70.5s\n",
      "train \t 165: 0.0013 100.0% 70.5s\n",
      "train \t 166: 0.0010 100.0% 70.5s\n",
      "train \t 167: 0.0010 100.0% 70.9s\n",
      "train \t 168: 0.0012 100.0% 70.5s\n",
      "train \t 169: 0.0007 100.0% 70.5s\n",
      "train \t 170: 0.0010 100.0% 70.5s\n",
      "train \t 171: 0.0011 100.0% 70.5s\n",
      "train \t 172: 0.0011 100.0% 70.5s\n",
      "train \t 173: 0.0011 100.0% 70.5s\n",
      "train \t 174: 0.0008 100.0% 70.5s\n",
      "train \t 175: 0.0009 100.0% 70.5s\n",
      "train \t 176: 0.0011 100.0% 70.5s\n",
      "train \t 177: 0.0009 100.0% 70.5s\n",
      "train \t 178: 0.0010 100.0% 70.5s\n",
      "train \t 179: 0.0010 100.0% 70.5s\n",
      "train \t 180: 0.0011 100.0% 70.5s\n",
      "train \t 181: 0.0010 100.0% 70.5s\n",
      "train \t 182: 0.0010 100.0% 70.5s\n",
      "train \t 183: 0.0012 100.0% 70.5s\n",
      "train \t 184: 0.0007 100.0% 70.5s\n",
      "train \t 185: 0.0012 100.0% 70.5s\n",
      "train \t 186: 0.0008 100.0% 70.5s\n",
      "train \t 187: 0.0011 100.0% 70.9s\n",
      "train \t 188: 0.0009 100.0% 70.5s\n",
      "train \t 189: 0.0011 100.0% 70.9s\n",
      "train \t 190: 0.0007 100.0% 70.5s\n",
      "train \t 191: 0.0007 100.0% 70.5s\n",
      "train \t 192: 0.0011 100.0% 70.5s\n",
      "train \t 193: 0.0008 100.0% 70.5s\n",
      "train \t 194: 0.0009 100.0% 70.5s\n",
      "train \t 195: 0.0008 100.0% 70.5s\n",
      "train \t 196: 0.0013 100.0% 70.9s\n",
      "train \t 197: 0.0009 100.0% 70.5s\n",
      "train \t 198: 0.0008 100.0% 70.5s\n",
      "train \t 199: 0.0013 100.0% 70.5s\n",
      "train \t 200: 0.0008 100.0% 70.5s\n",
      "val \t 200: 0.4120 93.6% 4.9s\n",
      "adv FSM \t\t\t 200: 20.1622 15.3% 18.4s\n",
      "adv PGD-20 \t\t\t 200: 56.2116 0.6% 273.6s\n",
      "adv PGD-100 \t\t\t 200: 57.1002 0.5% 1348.4s\n",
      "adv CW-100 \t\t\t 200: 1.5535 0.4% 1318.4s\n",
      "Finished Training\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 2 replays------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 1.6671 38.1% 140.8s\n",
      "train \t 2: 1.3077 51.7% 140.7s\n",
      "train \t 3: 1.1319 58.6% 140.7s\n",
      "train \t 4: 1.0202 62.6% 140.7s\n",
      "train \t 5: 0.9462 65.2% 140.7s\n",
      "train \t 6: 0.8731 67.6% 140.7s\n",
      "train \t 7: 0.8161 69.6% 140.7s\n",
      "train \t 8: 0.7718 71.1% 140.7s\n",
      "train \t 9: 0.7366 72.3% 140.7s\n",
      "train \t 10: 0.6896 73.9% 140.7s\n",
      "train \t 11: 0.6638 74.9% 140.7s\n",
      "train \t 12: 0.6392 75.9% 140.7s\n",
      "train \t 13: 0.6045 76.9% 140.7s\n",
      "train \t 14: 0.5729 78.1% 140.7s\n",
      "train \t 15: 0.5407 79.2% 140.6s\n",
      "train \t 16: 0.5204 79.9% 141.0s\n",
      "train \t 17: 0.4868 81.1% 140.6s\n",
      "train \t 18: 0.4742 81.6% 140.6s\n",
      "train \t 19: 0.4495 82.5% 140.6s\n",
      "train \t 20: 0.4274 83.3% 140.7s\n",
      "val \t 20: 0.3787 86.8% 4.9s\n",
      "train \t 21: 0.4109 83.8% 140.6s\n",
      "train \t 22: 0.3806 84.9% 140.6s\n",
      "train \t 23: 0.3639 85.5% 140.6s\n",
      "train \t 24: 0.3358 86.4% 140.6s\n",
      "train \t 25: 0.3331 86.7% 140.7s\n",
      "train \t 26: 0.2986 87.9% 140.6s\n",
      "train \t 27: 0.2721 88.9% 140.6s\n",
      "train \t 28: 0.2631 89.3% 140.6s\n",
      "train \t 29: 0.2440 90.1% 140.5s\n",
      "train \t 30: 0.2253 90.7% 141.3s\n",
      "train \t 31: 0.1509 93.9% 140.5s\n",
      "train \t 32: 0.1281 94.7% 140.6s\n",
      "train \t 33: 0.1140 95.2% 140.6s\n",
      "train \t 34: 0.1067 95.6% 141.4s\n",
      "train \t 35: 0.1030 95.7% 140.6s\n",
      "train \t 36: 0.0963 95.9% 140.6s\n",
      "train \t 37: 0.0910 96.3% 140.6s\n",
      "train \t 38: 0.0849 96.5% 140.6s\n",
      "train \t 39: 0.0822 96.6% 140.6s\n",
      "train \t 40: 0.0816 96.6% 140.6s\n",
      "val \t 40: 0.3660 90.1% 4.9s\n",
      "train \t 41: 0.0793 96.7% 140.6s\n",
      "train \t 42: 0.0765 96.9% 140.7s\n",
      "train \t 43: 0.0740 96.9% 140.6s\n",
      "train \t 44: 0.0705 97.1% 140.6s\n",
      "train \t 45: 0.0664 97.3% 141.4s\n",
      "train \t 46: 0.0652 97.3% 140.5s\n",
      "train \t 47: 0.0651 97.4% 140.6s\n",
      "train \t 48: 0.0655 97.3% 140.6s\n",
      "train \t 49: 0.0639 97.4% 140.6s\n",
      "train \t 50: 0.0611 97.5% 140.6s\n",
      "train \t 51: 0.0584 97.7% 140.6s\n",
      "train \t 52: 0.0598 97.5% 140.5s\n",
      "train \t 53: 0.0567 97.7% 140.7s\n",
      "train \t 54: 0.0552 97.8% 140.6s\n",
      "train \t 55: 0.0534 97.8% 141.4s\n",
      "train \t 56: 0.0534 97.8% 140.5s\n",
      "train \t 57: 0.0520 97.9% 140.6s\n",
      "train \t 58: 0.0523 97.9% 140.6s\n",
      "train \t 59: 0.0505 98.0% 141.4s\n",
      "train \t 60: 0.0481 98.1% 140.6s\n",
      "val \t 60: 0.4275 89.8% 4.9s\n",
      "train \t 61: 0.0455 98.2% 140.6s\n",
      "train \t 62: 0.0419 98.3% 140.6s\n",
      "train \t 63: 0.0411 98.4% 140.6s\n",
      "train \t 64: 0.0404 98.4% 140.6s\n",
      "train \t 65: 0.0399 98.4% 141.3s\n",
      "train \t 66: 0.0381 98.5% 141.3s\n",
      "train \t 67: 0.0382 98.5% 140.5s\n",
      "train \t 68: 0.0384 98.5% 141.4s\n",
      "train \t 69: 0.0365 98.5% 141.3s\n",
      "train \t 70: 0.0364 98.6% 140.5s\n",
      "train \t 71: 0.0354 98.6% 141.4s\n",
      "train \t 72: 0.0366 98.5% 140.6s\n",
      "train \t 73: 0.0362 98.6% 140.6s\n",
      "train \t 74: 0.0355 98.6% 141.3s\n",
      "train \t 75: 0.0351 98.6% 141.3s\n",
      "train \t 76: 0.0352 98.6% 141.4s\n",
      "train \t 77: 0.0343 98.6% 140.6s\n",
      "train \t 78: 0.0353 98.6% 140.6s\n",
      "train \t 79: 0.0341 98.7% 140.6s\n",
      "train \t 80: 0.0334 98.7% 140.5s\n",
      "val \t 80: 0.4764 89.8% 4.9s\n",
      "train \t 81: 0.0327 98.8% 140.6s\n",
      "train \t 82: 0.0330 98.7% 140.5s\n",
      "train \t 83: 0.0338 98.7% 140.5s\n",
      "train \t 84: 0.0330 98.7% 140.5s\n",
      "train \t 85: 0.0321 98.8% 141.4s\n",
      "train \t 86: 0.0317 98.7% 140.5s\n",
      "train \t 87: 0.0333 98.7% 140.6s\n",
      "train \t 88: 0.0327 98.8% 140.5s\n",
      "train \t 89: 0.0317 98.8% 140.5s\n",
      "train \t 90: 0.0320 98.7% 140.5s\n",
      "train \t 91: 0.0328 98.7% 140.5s\n",
      "train \t 92: 0.0315 98.8% 140.5s\n",
      "train \t 93: 0.0321 98.8% 140.5s\n",
      "train \t 94: 0.0315 98.8% 140.5s\n",
      "train \t 95: 0.0311 98.8% 140.5s\n",
      "train \t 96: 0.0313 98.8% 140.5s\n",
      "train \t 97: 0.0317 98.8% 140.5s\n",
      "train \t 98: 0.0318 98.8% 141.3s\n",
      "train \t 99: 0.0315 98.8% 140.5s\n",
      "train \t 100: 0.0319 98.8% 140.5s\n",
      "val \t 100: 0.4756 90.0% 4.9s\n",
      "adv FSM \t\t\t 100: 4.7072 43.3% 18.4s\n",
      "adv PGD-20 \t\t\t 100: 7.7010 29.3% 273.2s\n",
      "adv PGD-100 \t\t\t 100: 7.7703 29.0% 1341.6s\n",
      "adv CW-100 \t\t\t 100: 0.9436 29.5% 1329.6s\n",
      "Finished Training\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 4 replays------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 1.8648 30.4% 281.0s\n",
      "train \t 2: 1.6066 39.7% 281.0s\n",
      "train \t 3: 1.4575 45.6% 281.0s\n",
      "train \t 4: 1.3523 49.6% 282.6s\n",
      "train \t 5: 1.2780 52.3% 282.7s\n",
      "train \t 6: 1.2157 54.6% 281.1s\n",
      "train \t 7: 1.1645 56.5% 281.0s\n",
      "train \t 8: 1.1165 58.1% 281.0s\n",
      "train \t 9: 1.0708 59.8% 281.0s\n",
      "train \t 10: 1.0339 61.0% 281.0s\n",
      "val \t 10: 0.6309 80.0% 4.9s\n",
      "train \t 11: 0.9942 62.5% 281.9s\n",
      "train \t 12: 0.9588 63.8% 281.0s\n",
      "train \t 13: 0.9268 64.7% 282.6s\n",
      "train \t 14: 0.8910 65.9% 281.0s\n",
      "train \t 15: 0.8610 67.0% 280.9s\n",
      "train \t 16: 0.7431 71.8% 280.8s\n",
      "train \t 17: 0.6938 73.5% 282.4s\n",
      "train \t 18: 0.6717 74.2% 280.8s\n",
      "train \t 19: 0.6462 75.1% 280.8s\n",
      "train \t 20: 0.6249 75.7% 280.7s\n",
      "val \t 20: 0.4170 86.3% 4.9s\n",
      "train \t 21: 0.6070 76.4% 280.8s\n",
      "train \t 22: 0.5855 77.1% 280.8s\n",
      "train \t 23: 0.5634 77.9% 280.8s\n",
      "train \t 24: 0.5461 78.5% 282.4s\n",
      "train \t 25: 0.5268 79.1% 280.8s\n",
      "train \t 26: 0.5055 79.8% 280.8s\n",
      "train \t 27: 0.4873 80.5% 280.9s\n",
      "train \t 28: 0.4710 81.1% 280.9s\n",
      "train \t 29: 0.4514 81.9% 280.9s\n",
      "train \t 30: 0.4348 82.5% 282.5s\n",
      "val \t 30: 0.4252 86.2% 4.9s\n",
      "train \t 31: 0.3729 85.3% 280.9s\n",
      "train \t 32: 0.3483 86.3% 280.8s\n",
      "train \t 33: 0.3316 86.8% 280.8s\n",
      "train \t 34: 0.3206 87.3% 282.4s\n",
      "train \t 35: 0.3112 87.6% 280.7s\n",
      "train \t 36: 0.3033 87.9% 282.3s\n",
      "train \t 37: 0.2941 88.3% 282.4s\n",
      "train \t 38: 0.2857 88.6% 280.8s\n",
      "train \t 39: 0.2787 88.9% 282.4s\n",
      "train \t 40: 0.2727 89.0% 282.0s\n",
      "val \t 40: 0.4157 86.7% 4.9s\n",
      "train \t 41: 0.2582 89.8% 280.9s\n",
      "train \t 42: 0.2518 90.1% 280.8s\n",
      "train \t 43: 0.2486 90.1% 280.9s\n",
      "train \t 44: 0.2468 90.2% 280.9s\n",
      "train \t 45: 0.2422 90.5% 280.8s\n",
      "train \t 46: 0.2428 90.3% 280.8s\n",
      "train \t 47: 0.2411 90.4% 280.9s\n",
      "train \t 48: 0.2392 90.5% 280.8s\n",
      "train \t 49: 0.2365 90.6% 282.4s\n",
      "train \t 50: 0.2344 90.8% 282.5s\n",
      "val \t 50: 0.4193 86.7% 4.9s\n",
      "adv FSM \t\t\t 50: 1.9880 51.0% 18.3s\n",
      "adv PGD-20 \t\t\t 50: 2.6086 41.4% 272.0s\n",
      "adv PGD-100 \t\t\t 50: 2.6205 41.3% 1344.0s\n",
      "adv CW-100 \t\t\t 50: 0.7647 42.2% 1336.7s\n",
      "Finished Training\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 10 replays------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 1.8595 30.7% 702.2s\n",
      "train \t 2: 1.6584 37.5% 702.3s\n",
      "train \t 3: 1.5260 42.5% 702.1s\n",
      "train \t 4: 1.4334 45.9% 703.3s\n",
      "val \t 4: 1.0009 64.8% 4.9s\n",
      "train \t 5: 1.3636 48.7% 705.9s\n",
      "train \t 6: 1.2993 50.6% 702.1s\n",
      "train \t 7: 1.2284 53.4% 701.8s\n",
      "train \t 8: 1.1791 55.0% 701.6s\n",
      "val \t 8: 0.7166 77.0% 4.9s\n",
      "train \t 9: 1.1478 56.1% 705.4s\n",
      "train \t 10: 1.1212 57.0% 702.1s\n",
      "train \t 11: 1.0951 57.8% 701.9s\n",
      "train \t 12: 1.0667 59.0% 702.0s\n",
      "val \t 12: 0.6261 80.0% 4.9s\n",
      "train \t 13: 1.0278 60.4% 701.7s\n",
      "train \t 14: 1.0041 61.1% 705.1s\n",
      "train \t 15: 0.9909 61.7% 702.0s\n",
      "train \t 16: 0.9782 62.1% 702.1s\n",
      "val \t 16: 0.5726 81.9% 4.9s\n",
      "train \t 17: 0.9647 62.6% 701.9s\n",
      "train \t 18: 0.9569 62.9% 701.8s\n",
      "train \t 19: 0.9516 63.2% 705.8s\n",
      "train \t 20: 0.9496 63.3% 701.7s\n",
      "val \t 20: 0.5592 82.5% 4.9s\n",
      "adv FSM \t\t\t 20: 1.2712 52.9% 18.1s\n",
      "adv PGD-20 \t\t\t 20: 1.4259 47.1% 272.7s\n",
      "adv PGD-100 \t\t\t 20: 1.4289 47.0% 1339.0s\n",
      "adv CW-100 \t\t\t 20: 0.8245 46.4% 1337.8s\n"
     ]
    }
   ],
   "source": [
    "free_logs = defaultdict(lambda : defaultdict(lambda :[]))\n",
    "\n",
    "for K in free_Ks:\n",
    "    print(f'\\n\\n\\n\\n\\ntraining with {K} replays------------------------\\n\\n\\n\\n')\n",
    "    model, optimizer, scheduler = build_model(K=K)\n",
    "    \n",
    "    for epoch in range(int(EPOCHS / K)):  # loop over the dataset multiple times\n",
    "        logs = train_with_replay(K, model, trainloader, optimizer, epoch,\n",
    "                                after_func=lambda model: model.adv.step())\n",
    "        free_logs[K]['train'].append(logs)\n",
    "        \n",
    "        scheduler.step()\n",
    "        if (epoch * K + K) % TEST_EVERY == 0:\n",
    "\n",
    "            logs = run_val(model, testloader, epoch)\n",
    "            free_logs[K]['test'].append(logs)\n",
    "\n",
    "            # adv loss\n",
    "    run_attacks(free_logs[K], attacks, attack_names, model, testloader, epoch)\n",
    "    \n",
    "\n",
    "    print('Finished Training')\n",
    "    torch.save(model.state_dict(), f\"snapshots/wresnet-cifar-10-free-{K}.pch\")\n",
    "    del model\n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "with open('snapshots/free_logs.pickle', 'wb') as fd:\n",
    "    pickle.dump(holder_to_dict(free_logs), fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 1 replays------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 1.4033 48.6% 70.5s\n",
      "train \t 2: 0.8623 69.6% 70.5s\n",
      "train \t 3: 0.6493 77.6% 70.7s\n",
      "train \t 4: 0.5334 81.4% 70.5s\n",
      "train \t 5: 0.4524 84.2% 70.5s\n",
      "train \t 6: 0.4021 86.0% 70.5s\n",
      "train \t 7: 0.3554 87.7% 70.5s\n",
      "train \t 8: 0.3164 89.2% 70.5s\n",
      "train \t 9: 0.2872 90.1% 70.5s\n",
      "train \t 10: 0.2611 90.9% 70.5s\n",
      "train \t 11: 0.2374 91.8% 70.5s\n",
      "train \t 12: 0.2204 92.2% 70.5s\n",
      "train \t 13: 0.1986 93.0% 70.5s\n",
      "train \t 14: 0.1839 93.5% 70.5s\n",
      "train \t 15: 0.1719 93.9% 70.5s\n",
      "train \t 16: 0.1539 94.6% 70.4s\n",
      "train \t 17: 0.1418 95.1% 70.4s\n",
      "train \t 18: 0.1302 95.4% 70.8s\n",
      "train \t 19: 0.1231 95.7% 70.4s\n",
      "train \t 20: 0.1162 95.8% 70.4s\n",
      "train \t 21: 0.1056 96.3% 70.6s\n",
      "train \t 22: 0.0957 96.6% 70.5s\n",
      "train \t 23: 0.0918 96.7% 70.4s\n",
      "train \t 24: 0.0814 97.1% 70.4s\n",
      "train \t 25: 0.0770 97.3% 70.4s\n",
      "train \t 26: 0.0726 97.4% 70.4s\n",
      "train \t 27: 0.0656 97.7% 70.4s\n",
      "train \t 28: 0.0610 97.9% 70.4s\n",
      "train \t 29: 0.0594 97.9% 70.4s\n",
      "train \t 30: 0.0519 98.2% 70.4s\n",
      "train \t 31: 0.0539 98.1% 70.4s\n",
      "train \t 32: 0.0484 98.3% 70.4s\n",
      "train \t 33: 0.0427 98.6% 70.4s\n",
      "train \t 34: 0.0404 98.6% 70.4s\n",
      "train \t 35: 0.0453 98.4% 70.5s\n",
      "train \t 36: 0.0410 98.5% 70.4s\n",
      "train \t 37: 0.0373 98.7% 70.6s\n",
      "train \t 38: 0.0340 98.8% 70.9s\n",
      "train \t 39: 0.0333 98.8% 70.4s\n",
      "train \t 40: 0.0352 98.8% 70.4s\n",
      "val \t 40: 0.4144 91.5% 4.9s\n",
      "train \t 41: 0.0291 99.0% 70.4s\n",
      "train \t 42: 0.0294 99.0% 70.8s\n",
      "train \t 43: 0.0273 99.1% 70.8s\n",
      "train \t 44: 0.0281 99.0% 70.4s\n",
      "train \t 45: 0.0256 99.1% 70.4s\n",
      "train \t 46: 0.0254 99.1% 70.4s\n",
      "train \t 47: 0.0237 99.2% 70.8s\n",
      "train \t 48: 0.0192 99.4% 70.4s\n",
      "train \t 49: 0.0240 99.2% 70.6s\n",
      "train \t 50: 0.0210 99.3% 70.6s\n",
      "train \t 51: 0.0224 99.2% 70.6s\n",
      "train \t 52: 0.0175 99.4% 70.4s\n",
      "train \t 53: 0.0198 99.3% 70.4s\n",
      "train \t 54: 0.0156 99.5% 70.7s\n",
      "train \t 55: 0.0169 99.5% 70.5s\n",
      "train \t 56: 0.0171 99.4% 70.6s\n",
      "train \t 57: 0.0147 99.5% 70.6s\n",
      "train \t 58: 0.0167 99.4% 70.8s\n",
      "train \t 59: 0.0157 99.4% 70.6s\n",
      "train \t 60: 0.0159 99.4% 70.6s\n",
      "train \t 61: 0.0072 99.8% 70.6s\n",
      "train \t 62: 0.0034 99.9% 70.5s\n",
      "train \t 63: 0.0028 99.9% 70.6s\n",
      "train \t 64: 0.0025 99.9% 70.6s\n",
      "train \t 65: 0.0022 99.9% 70.6s\n",
      "train \t 66: 0.0021 99.9% 70.6s\n",
      "train \t 67: 0.0018 100.0% 70.6s\n",
      "train \t 68: 0.0018 100.0% 70.7s\n",
      "train \t 69: 0.0015 100.0% 70.5s\n",
      "train \t 70: 0.0014 100.0% 70.6s\n",
      "train \t 71: 0.0013 100.0% 70.5s\n",
      "train \t 72: 0.0014 100.0% 70.5s\n",
      "train \t 73: 0.0014 100.0% 70.4s\n",
      "train \t 74: 0.0014 100.0% 70.7s\n",
      "train \t 75: 0.0011 100.0% 70.4s\n",
      "train \t 76: 0.0011 100.0% 70.4s\n",
      "train \t 77: 0.0010 100.0% 70.4s\n",
      "train \t 78: 0.0008 100.0% 70.4s\n",
      "train \t 79: 0.0010 100.0% 70.4s\n",
      "train \t 80: 0.0010 100.0% 70.4s\n",
      "val \t 80: 0.3551 94.0% 4.9s\n",
      "train \t 81: 0.0009 100.0% 70.7s\n",
      "train \t 82: 0.0010 100.0% 70.4s\n",
      "train \t 83: 0.0009 100.0% 70.4s\n",
      "train \t 84: 0.0012 100.0% 70.4s\n",
      "train \t 85: 0.0008 100.0% 70.4s\n",
      "train \t 86: 0.0007 100.0% 70.4s\n",
      "train \t 87: 0.0008 100.0% 70.4s\n",
      "train \t 88: 0.0005 100.0% 70.4s\n",
      "train \t 89: 0.0005 100.0% 70.4s\n",
      "train \t 90: 0.0005 100.0% 70.4s\n",
      "train \t 91: 0.0006 100.0% 70.4s\n",
      "train \t 92: 0.0007 100.0% 70.4s\n",
      "train \t 93: 0.0006 100.0% 70.8s\n",
      "train \t 94: 0.0007 100.0% 70.4s\n",
      "train \t 95: 0.0005 100.0% 70.4s\n",
      "train \t 96: 0.0006 100.0% 70.5s\n",
      "train \t 97: 0.0006 100.0% 70.5s\n",
      "train \t 98: 0.0005 100.0% 70.4s\n",
      "train \t 99: 0.0007 100.0% 70.4s\n",
      "train \t 100: 0.0007 100.0% 70.4s\n",
      "train \t 101: 0.0005 100.0% 70.4s\n",
      "train \t 102: 0.0004 100.0% 70.4s\n",
      "train \t 103: 0.0007 100.0% 70.4s\n",
      "train \t 104: 0.0005 100.0% 70.4s\n",
      "train \t 105: 0.0006 100.0% 70.4s\n",
      "train \t 106: 0.0006 100.0% 70.4s\n",
      "train \t 107: 0.0005 100.0% 70.5s\n",
      "train \t 108: 0.0005 100.0% 70.4s\n",
      "train \t 109: 0.0004 100.0% 70.4s\n",
      "train \t 110: 0.0004 100.0% 70.4s\n",
      "train \t 111: 0.0004 100.0% 70.4s\n",
      "train \t 112: 0.0004 100.0% 70.4s\n",
      "train \t 113: 0.0005 100.0% 70.4s\n",
      "train \t 114: 0.0004 100.0% 70.4s\n",
      "train \t 115: 0.0005 100.0% 70.4s\n",
      "train \t 116: 0.0004 100.0% 70.4s\n",
      "train \t 117: 0.0004 100.0% 70.4s\n",
      "train \t 118: 0.0006 100.0% 70.4s\n",
      "train \t 119: 0.0005 100.0% 70.4s\n",
      "train \t 120: 0.0004 100.0% 70.4s\n",
      "val \t 120: 0.3867 94.0% 4.9s\n",
      "train \t 121: 0.0003 100.0% 70.4s\n",
      "train \t 122: 0.0005 100.0% 70.5s\n",
      "train \t 123: 0.0004 100.0% 70.8s\n",
      "train \t 124: 0.0004 100.0% 70.7s\n",
      "train \t 125: 0.0005 100.0% 70.4s\n",
      "train \t 126: 0.0003 100.0% 70.4s\n",
      "train \t 127: 0.0004 100.0% 70.4s\n",
      "train \t 128: 0.0004 100.0% 70.4s\n",
      "train \t 129: 0.0003 100.0% 70.4s\n",
      "train \t 130: 0.0004 100.0% 70.5s\n",
      "train \t 131: 0.0005 100.0% 70.5s\n",
      "train \t 132: 0.0004 100.0% 70.6s\n",
      "train \t 133: 0.0003 100.0% 70.4s\n",
      "train \t 134: 0.0004 100.0% 70.8s\n",
      "train \t 135: 0.0004 100.0% 70.8s\n",
      "train \t 136: 0.0003 100.0% 70.4s\n",
      "train \t 137: 0.0003 100.0% 70.8s\n",
      "train \t 138: 0.0003 100.0% 70.4s\n",
      "train \t 139: 0.0004 100.0% 70.4s\n",
      "train \t 140: 0.0005 100.0% 70.4s\n",
      "train \t 141: 0.0002 100.0% 70.4s\n",
      "train \t 142: 0.0003 100.0% 70.4s\n",
      "train \t 143: 0.0003 100.0% 70.4s\n",
      "train \t 144: 0.0003 100.0% 70.4s\n",
      "train \t 145: 0.0003 100.0% 70.4s\n",
      "train \t 146: 0.0004 100.0% 70.4s\n",
      "train \t 147: 0.0004 100.0% 70.4s\n",
      "train \t 148: 0.0003 100.0% 70.7s\n",
      "train \t 149: 0.0003 100.0% 70.4s\n",
      "train \t 150: 0.0003 100.0% 70.4s\n",
      "train \t 151: 0.0003 100.0% 70.4s\n",
      "train \t 152: 0.0002 100.0% 70.5s\n",
      "train \t 153: 0.0003 100.0% 70.4s\n",
      "train \t 154: 0.0003 100.0% 70.4s\n",
      "train \t 155: 0.0003 100.0% 70.4s\n",
      "train \t 156: 0.0003 100.0% 70.4s\n",
      "train \t 157: 0.0004 100.0% 70.4s\n",
      "train \t 158: 0.0003 100.0% 70.4s\n",
      "train \t 159: 0.0003 100.0% 70.4s\n",
      "train \t 160: 0.0003 100.0% 70.4s\n",
      "val \t 160: 0.3841 93.9% 4.9s\n",
      "train \t 161: 0.0002 100.0% 70.4s\n",
      "train \t 162: 0.0003 100.0% 70.4s\n",
      "train \t 163: 0.0003 100.0% 70.4s\n",
      "train \t 164: 0.0003 100.0% 70.5s\n",
      "train \t 165: 0.0003 100.0% 70.4s\n",
      "train \t 166: 0.0002 100.0% 70.4s\n",
      "train \t 167: 0.0002 100.0% 70.4s\n",
      "train \t 168: 0.0003 100.0% 70.7s\n",
      "train \t 169: 0.0002 100.0% 70.4s\n",
      "train \t 170: 0.0003 100.0% 70.4s\n",
      "train \t 171: 0.0003 100.0% 70.4s\n",
      "train \t 172: 0.0002 100.0% 70.4s\n",
      "train \t 173: 0.0002 100.0% 70.8s\n",
      "train \t 174: 0.0003 100.0% 70.4s\n",
      "train \t 175: 0.0003 100.0% 70.8s\n",
      "train \t 176: 0.0003 100.0% 70.4s\n",
      "train \t 177: 0.0002 100.0% 70.4s\n",
      "train \t 178: 0.0004 100.0% 70.4s\n",
      "train \t 179: 0.0002 100.0% 70.4s\n",
      "train \t 180: 0.0003 100.0% 70.4s\n",
      "train \t 181: 0.0002 100.0% 70.4s\n",
      "train \t 182: 0.0004 100.0% 70.4s\n",
      "train \t 183: 0.0003 100.0% 70.8s\n",
      "train \t 184: 0.0003 100.0% 70.4s\n",
      "train \t 185: 0.0003 100.0% 70.8s\n",
      "train \t 186: 0.0003 100.0% 70.4s\n",
      "train \t 187: 0.0003 100.0% 70.8s\n",
      "train \t 188: 0.0004 100.0% 70.4s\n",
      "train \t 189: 0.0002 100.0% 70.4s\n",
      "train \t 190: 0.0002 100.0% 70.8s\n",
      "train \t 191: 0.0002 100.0% 70.4s\n",
      "train \t 192: 0.0003 100.0% 70.4s\n",
      "train \t 193: 0.0003 100.0% 70.4s\n",
      "train \t 194: 0.0003 100.0% 70.4s\n",
      "train \t 195: 0.0003 100.0% 70.4s\n",
      "train \t 196: 0.0004 100.0% 70.4s\n",
      "train \t 197: 0.0003 100.0% 70.8s\n",
      "train \t 198: 0.0003 100.0% 70.4s\n",
      "train \t 199: 0.0003 100.0% 70.4s\n",
      "train \t 200: 0.0003 100.0% 70.4s\n",
      "val \t 200: 0.3846 94.0% 4.9s\n",
      "adv FSM \t\t\t 200: 22.1568 10.8% 18.3s\n",
      "adv PGD-20 \t\t\t 200: 99.5745 0.0% 271.8s\n",
      "adv PGD-100 \t\t\t 200: 109.6868 0.0% 1339.2s\n",
      "adv CW-100 \t\t\t 200: 3.0615 0.0% 1311.3s\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "#standard training with replay logs\n",
    "srl = defaultdict(lambda : defaultdict(lambda : []))\n",
    "for K in [training_with_replay_Ks[0]]:\n",
    "    print(f'\\n\\n\\n\\n\\ntraining with {K} replays------------------------\\n\\n\\n\\n')\n",
    "\n",
    "    model, optimizer, scheduler = build_model(False, K=K)\n",
    "        \n",
    "    for epoch in range(int(EPOCHS / K)): # loop over the dataset multiple times\n",
    "            \n",
    "        logs = train_with_replay(K, model, trainloader, optimizer, epoch)\n",
    "        \n",
    "        scheduler.step()\n",
    "        srl[K]['train'].append(logs)\n",
    "        if (epoch * K + K) % TEST_EVERY == 0:\n",
    "            # valdiation loss\n",
    "            logs = run_val(model, testloader, epoch)\n",
    "            srl[K]['test'].append(logs)\n",
    "    run_attacks(srl[K], attacks, \n",
    "                attack_names, model, testloader, epoch)\n",
    "    print('Finished Training')\n",
    "    torch.save(model.state_dict(), f\"wresnet-cifar-10-normal-{K}.pch\")\n",
    "    del model\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "with open('snapshots/srl.pickle', 'wb') as fd:\n",
    "    pickle.dump(holder_to_dict(srl), fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 1-PGD------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 2.0247 23.6% 136.0s\n",
      "train \t 2: 1.8348 30.4% 137.3s\n",
      "train \t 3: 1.7204 34.4% 138.2s\n",
      "train \t 4: 1.6313 38.0% 137.5s\n",
      "train \t 5: 1.5662 40.5% 137.5s\n",
      "train \t 6: 1.5104 42.6% 137.5s\n",
      "train \t 7: 1.4617 44.6% 137.5s\n",
      "train \t 8: 1.4238 46.2% 137.5s\n",
      "train \t 9: 1.3879 47.4% 137.5s\n",
      "train \t 10: 1.3541 48.6% 137.6s\n",
      "train \t 11: 1.3222 49.9% 137.6s\n",
      "train \t 12: 1.2946 50.7% 137.5s\n",
      "train \t 13: 1.2669 51.8% 137.5s\n",
      "train \t 14: 1.2390 52.7% 137.5s\n",
      "train \t 15: 1.2097 53.7% 137.5s\n",
      "train \t 16: 1.1880 54.7% 137.5s\n",
      "train \t 17: 1.1607 55.7% 137.5s\n",
      "train \t 18: 0.9996 61.9% 137.5s\n",
      "train \t 19: 0.4728 83.3% 137.6s\n",
      "train \t 20: 0.3548 87.6% 137.6s\n",
      "train \t 21: 0.2962 89.7% 137.5s\n",
      "train \t 22: 0.2675 90.7% 137.6s\n",
      "train \t 23: 0.2267 92.1% 137.5s\n",
      "train \t 24: 0.2015 93.1% 137.5s\n",
      "train \t 25: 0.1866 93.4% 137.6s\n",
      "train \t 26: 0.1765 93.9% 137.5s\n",
      "train \t 27: 0.1535 94.6% 138.2s\n",
      "train \t 28: 0.1460 94.9% 137.5s\n",
      "train \t 29: 0.1162 96.0% 137.5s\n",
      "train \t 30: 0.1097 96.1% 137.5s\n",
      "train \t 31: 0.0958 96.6% 137.4s\n",
      "train \t 32: 0.0862 97.1% 137.5s\n",
      "train \t 33: 0.0780 97.3% 137.4s\n",
      "train \t 34: 0.0739 97.4% 137.5s\n",
      "train \t 35: 0.0555 98.1% 137.4s\n",
      "train \t 36: 0.0447 98.5% 138.2s\n",
      "train \t 37: 0.0441 98.5% 137.5s\n",
      "train \t 38: 0.0347 98.8% 137.5s\n",
      "train \t 39: 0.0449 98.5% 137.5s\n",
      "train \t 40: 0.0351 98.8% 137.5s\n",
      "val \t 40: 0.8210 80.6% 4.9s\n",
      "train \t 41: 0.0390 98.7% 137.5s\n",
      "train \t 42: 0.0304 98.9% 137.5s\n",
      "train \t 43: 0.0282 99.0% 137.5s\n",
      "train \t 44: 0.0382 98.7% 137.5s\n",
      "train \t 45: 0.0307 99.0% 137.5s\n",
      "train \t 46: 0.0357 98.8% 137.5s\n",
      "train \t 47: 0.0293 99.0% 137.4s\n",
      "train \t 48: 0.0332 98.9% 137.4s\n",
      "train \t 49: 0.0378 98.7% 137.4s\n",
      "train \t 50: 0.0355 98.8% 137.4s\n",
      "train \t 51: 0.0307 99.0% 137.4s\n",
      "train \t 52: 0.0304 98.9% 137.4s\n",
      "train \t 53: 0.0316 98.9% 137.4s\n",
      "train \t 54: 0.0322 98.9% 137.5s\n",
      "train \t 55: 0.0262 99.1% 137.4s\n",
      "train \t 56: 0.0227 99.2% 138.2s\n",
      "train \t 57: 0.0299 99.0% 137.5s\n",
      "train \t 58: 0.0208 99.3% 137.4s\n",
      "train \t 59: 0.0222 99.3% 137.4s\n",
      "train \t 60: 0.0264 99.1% 137.5s\n",
      "train \t 61: 0.0124 99.6% 137.4s\n",
      "train \t 62: 0.0076 99.8% 137.4s\n",
      "train \t 63: 0.0054 99.8% 137.4s\n",
      "train \t 64: 0.0045 99.9% 137.5s\n",
      "train \t 65: 0.0048 99.9% 137.5s\n",
      "train \t 66: 0.0036 99.9% 137.5s\n",
      "train \t 67: 0.0041 99.9% 137.4s\n",
      "train \t 68: 0.0036 99.9% 137.4s\n",
      "train \t 69: 0.0037 99.9% 137.4s\n",
      "train \t 70: 0.0031 99.9% 137.4s\n",
      "train \t 71: 0.0034 99.9% 137.4s\n",
      "train \t 72: 0.0032 99.9% 137.5s\n",
      "train \t 73: 0.0030 99.9% 137.5s\n",
      "train \t 74: 0.0031 99.9% 137.4s\n",
      "train \t 75: 0.0021 99.9% 137.4s\n",
      "train \t 76: 0.0037 99.9% 137.4s\n",
      "train \t 77: 0.0029 99.9% 137.4s\n",
      "train \t 78: 0.0021 100.0% 137.4s\n",
      "train \t 79: 0.0024 99.9% 137.4s\n",
      "train \t 80: 0.0019 99.9% 137.4s\n",
      "val \t 80: 0.5885 88.6% 4.9s\n",
      "train \t 81: 0.0022 99.9% 137.4s\n",
      "train \t 82: 0.0025 99.9% 137.5s\n",
      "train \t 83: 0.0023 99.9% 138.1s\n",
      "train \t 84: 0.0023 99.9% 137.4s\n",
      "train \t 85: 0.0023 99.9% 138.1s\n",
      "train \t 86: 0.0018 99.9% 137.4s\n",
      "train \t 87: 0.0021 99.9% 137.4s\n",
      "train \t 88: 0.0017 99.9% 137.4s\n",
      "train \t 89: 0.0018 100.0% 137.4s\n",
      "train \t 90: 0.0015 100.0% 137.4s\n",
      "train \t 91: 0.0017 99.9% 137.4s\n",
      "train \t 92: 0.0020 99.9% 137.4s\n",
      "train \t 93: 0.0018 100.0% 137.4s\n",
      "train \t 94: 0.0015 100.0% 137.4s\n",
      "train \t 95: 0.0017 100.0% 137.4s\n",
      "train \t 96: 0.0024 99.9% 137.4s\n",
      "train \t 97: 0.0013 100.0% 137.4s\n",
      "train \t 98: 0.0018 99.9% 137.4s\n",
      "train \t 99: 0.0015 100.0% 137.4s\n",
      "train \t 100: 0.0013 100.0% 137.4s\n",
      "val \t 100: 0.6510 88.1% 4.9s\n",
      "adv FSM \t\t\t 100: 0.1047 97.7% 18.3s\n",
      "adv PGD-20 \t\t\t 100: 81.9542 0.0% 273.3s\n",
      "adv PGD-100 \t\t\t 100: 96.9064 0.0% 1341.0s\n",
      "adv CW-100 \t\t\t 100: 3.5464 0.0% 1311.3s\n",
      "train \t 101: 0.0014 100.0% 137.4s\n",
      "train \t 102: 0.0011 100.0% 137.4s\n",
      "train \t 103: 0.0011 100.0% 137.4s\n",
      "train \t 104: 0.0016 99.9% 137.5s\n",
      "train \t 105: 0.0012 100.0% 137.4s\n",
      "train \t 106: 0.0017 100.0% 137.4s\n",
      "train \t 107: 0.0012 100.0% 137.4s\n",
      "train \t 108: 0.0014 100.0% 137.4s\n",
      "train \t 109: 0.0012 100.0% 137.4s\n",
      "train \t 110: 0.0015 100.0% 137.5s\n",
      "train \t 111: 0.0014 100.0% 137.4s\n",
      "train \t 112: 0.0012 100.0% 137.4s\n",
      "train \t 113: 0.0016 99.9% 137.4s\n",
      "train \t 114: 0.0013 100.0% 137.4s\n",
      "train \t 115: 0.0011 100.0% 137.4s\n",
      "train \t 116: 0.0012 100.0% 137.4s\n",
      "train \t 117: 0.0012 100.0% 138.0s\n",
      "train \t 118: 0.0010 100.0% 137.4s\n",
      "train \t 119: 0.0012 100.0% 137.4s\n",
      "train \t 120: 0.0011 100.0% 137.4s\n",
      "val \t 120: 0.6549 88.6% 4.9s\n",
      "train \t 121: 0.0010 100.0% 137.4s\n",
      "train \t 122: 0.0014 100.0% 137.4s\n",
      "train \t 123: 0.0012 100.0% 137.4s\n",
      "train \t 124: 0.0007 100.0% 137.4s\n",
      "train \t 125: 0.0013 100.0% 137.4s\n",
      "train \t 126: 0.0011 100.0% 138.1s\n",
      "train \t 127: 0.0011 100.0% 137.4s\n",
      "train \t 128: 0.0009 100.0% 137.4s\n",
      "train \t 129: 0.0009 100.0% 137.4s\n",
      "train \t 130: 0.0008 100.0% 137.4s\n",
      "train \t 131: 0.0009 100.0% 137.4s\n",
      "train \t 132: 0.0009 100.0% 137.4s\n",
      "train \t 133: 0.0010 100.0% 137.4s\n",
      "train \t 134: 0.0008 100.0% 137.4s\n",
      "train \t 135: 0.0010 100.0% 137.4s\n",
      "train \t 136: 0.0007 100.0% 137.4s\n",
      "train \t 137: 0.0007 100.0% 137.4s\n",
      "train \t 138: 0.0010 100.0% 137.4s\n",
      "train \t 139: 0.0008 100.0% 137.4s\n",
      "train \t 140: 0.0010 100.0% 137.4s\n",
      "train \t 141: 0.0010 100.0% 137.4s\n",
      "train \t 142: 0.0010 100.0% 137.4s\n",
      "train \t 143: 0.0009 100.0% 137.4s\n",
      "train \t 144: 0.0009 100.0% 137.4s\n",
      "train \t 145: 0.0009 100.0% 137.4s\n",
      "train \t 146: 0.0008 100.0% 137.4s\n",
      "train \t 147: 0.0007 100.0% 137.4s\n",
      "train \t 148: 0.0010 100.0% 137.4s\n",
      "train \t 149: 0.0008 100.0% 137.4s\n",
      "train \t 150: 0.0009 100.0% 137.5s\n",
      "train \t 151: 0.0008 100.0% 137.4s\n",
      "train \t 152: 0.0008 100.0% 137.4s\n",
      "train \t 153: 0.0008 100.0% 137.4s\n",
      "train \t 154: 0.0007 100.0% 137.4s\n",
      "train \t 155: 0.0008 100.0% 137.5s\n",
      "train \t 156: 0.0006 100.0% 137.4s\n",
      "train \t 157: 0.0008 100.0% 137.4s\n",
      "train \t 158: 0.0008 100.0% 137.5s\n",
      "train \t 159: 0.0009 100.0% 137.5s\n",
      "train \t 160: 0.0009 100.0% 137.4s\n",
      "val \t 160: 0.6658 88.6% 4.9s\n",
      "train \t 161: 0.0007 100.0% 137.5s\n",
      "train \t 162: 0.0008 100.0% 137.5s\n",
      "train \t 163: 0.0008 100.0% 137.5s\n",
      "train \t 164: 0.0010 100.0% 137.5s\n",
      "train \t 165: 0.0007 100.0% 137.4s\n",
      "train \t 166: 0.0006 100.0% 137.5s\n",
      "train \t 167: 0.0008 100.0% 137.5s\n",
      "train \t 168: 0.0008 100.0% 137.4s\n",
      "train \t 169: 0.0006 100.0% 137.4s\n",
      "train \t 170: 0.0008 100.0% 137.4s\n",
      "train \t 171: 0.0007 100.0% 137.4s\n",
      "train \t 172: 0.0008 100.0% 137.4s\n",
      "train \t 173: 0.0008 100.0% 137.4s\n",
      "train \t 174: 0.0008 100.0% 137.5s\n",
      "train \t 175: 0.0008 100.0% 137.5s\n",
      "train \t 176: 0.0009 100.0% 137.5s\n",
      "train \t 177: 0.0011 100.0% 137.5s\n",
      "train \t 178: 0.0007 100.0% 137.5s\n",
      "train \t 179: 0.0011 100.0% 137.4s\n",
      "train \t 180: 0.0007 100.0% 137.4s\n",
      "train \t 181: 0.0007 100.0% 137.4s\n",
      "train \t 182: 0.0007 100.0% 137.4s\n",
      "train \t 183: 0.0007 100.0% 137.4s\n",
      "train \t 184: 0.0007 100.0% 137.4s\n",
      "train \t 185: 0.0006 100.0% 137.4s\n",
      "train \t 186: 0.0010 100.0% 137.4s\n",
      "train \t 187: 0.0007 100.0% 137.4s\n",
      "train \t 188: 0.0007 100.0% 137.4s\n",
      "train \t 189: 0.0006 100.0% 137.4s\n",
      "train \t 190: 0.0009 100.0% 137.4s\n",
      "train \t 191: 0.0008 100.0% 137.4s\n",
      "train \t 192: 0.0010 100.0% 137.5s\n",
      "train \t 193: 0.0008 100.0% 137.4s\n",
      "train \t 194: 0.0009 100.0% 137.5s\n",
      "train \t 195: 0.0005 100.0% 137.4s\n",
      "train \t 196: 0.0008 100.0% 137.4s\n",
      "train \t 197: 0.0006 100.0% 137.4s\n",
      "train \t 198: 0.0005 100.0% 137.4s\n",
      "train \t 199: 0.0007 100.0% 137.4s\n",
      "train \t 200: 0.0007 100.0% 137.4s\n",
      "val \t 200: 0.6594 88.5% 4.9s\n",
      "adv FSM \t\t\t 200: 0.1198 97.6% 18.3s\n",
      "adv PGD-20 \t\t\t 200: 83.8218 0.0% 272.2s\n",
      "adv PGD-100 \t\t\t 200: 99.1485 0.0% 1340.7s\n",
      "adv CW-100 \t\t\t 200: 3.6697 0.0% 1311.3s\n",
      "Finished Training\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "training with 2-PGD------------------------\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "train \t 1: 2.0510 23.3% 204.1s\n",
      "train \t 2: 1.8700 29.4% 204.2s\n",
      "train \t 3: 1.7601 33.5% 204.3s\n",
      "train \t 4: 1.6747 36.5% 204.3s\n",
      "train \t 5: 1.6115 39.2% 204.2s\n",
      "train \t 6: 1.5540 41.3% 204.3s\n",
      "train \t 7: 1.5068 42.9% 204.2s\n",
      "train \t 8: 1.4690 44.5% 204.9s\n",
      "train \t 9: 1.4353 45.6% 204.3s\n",
      "train \t 10: 1.4008 46.7% 204.2s\n",
      "train \t 11: 1.3734 47.8% 204.2s\n",
      "train \t 12: 1.3453 48.8% 204.9s\n",
      "train \t 13: 1.3186 49.6% 204.2s\n",
      "train \t 14: 1.2943 50.5% 204.2s\n",
      "train \t 15: 1.2703 51.2% 204.1s\n",
      "train \t 16: 1.2476 52.2% 204.2s\n",
      "train \t 17: 1.2247 53.0% 204.1s\n",
      "train \t 18: 1.2016 53.9% 204.2s\n",
      "train \t 19: 1.1790 54.5% 204.1s\n",
      "train \t 20: 1.1587 55.4% 204.1s\n",
      "train \t 21: 1.1376 56.1% 204.1s\n",
      "train \t 22: 1.1165 56.5% 204.1s\n",
      "train \t 23: 1.0988 57.3% 204.1s\n",
      "train \t 24: 1.0721 58.3% 204.1s\n",
      "train \t 25: 1.0528 58.9% 204.1s\n",
      "train \t 26: 1.0323 59.7% 205.0s\n",
      "train \t 27: 1.0119 60.2% 204.1s\n",
      "train \t 28: 0.9929 60.9% 204.1s\n",
      "train \t 29: 0.9722 61.7% 204.1s\n",
      "train \t 30: 0.9493 62.4% 204.9s\n",
      "train \t 31: 0.9308 63.1% 204.0s\n",
      "train \t 32: 0.9093 63.8% 204.0s\n",
      "train \t 33: 0.8870 64.6% 204.0s\n",
      "train \t 34: 0.8666 65.2% 204.0s\n",
      "train \t 35: 0.8465 66.1% 204.1s\n",
      "train \t 36: 0.8241 66.6% 204.1s\n",
      "train \t 37: 0.8085 67.3% 204.1s\n",
      "train \t 38: 0.7859 68.1% 204.1s\n",
      "train \t 39: 0.7692 68.7% 204.1s\n",
      "train \t 40: 0.7485 69.3% 204.1s\n",
      "val \t 40: 0.5167 83.3% 4.9s\n",
      "train \t 41: 0.7289 70.1% 204.0s\n",
      "train \t 42: 0.7122 70.8% 204.0s\n",
      "train \t 43: 0.6913 71.5% 204.0s\n",
      "train \t 44: 0.6761 72.0% 204.1s\n",
      "train \t 45: 0.6571 72.7% 204.0s\n",
      "train \t 46: 0.6453 73.2% 204.0s\n",
      "train \t 47: 0.6165 74.4% 203.9s\n",
      "train \t 48: 0.6030 74.8% 203.9s\n",
      "train \t 49: 0.5922 75.6% 204.0s\n",
      "train \t 50: 0.5741 76.1% 203.9s\n",
      "train \t 51: 0.5542 76.7% 203.9s\n",
      "train \t 52: 0.5457 77.2% 204.0s\n",
      "train \t 53: 0.5271 77.7% 204.0s\n",
      "train \t 54: 0.5118 78.5% 204.0s\n",
      "train \t 55: 0.4978 79.1% 203.9s\n",
      "train \t 56: 0.4823 79.7% 203.9s\n",
      "train \t 57: 0.4672 80.2% 203.9s\n",
      "train \t 58: 0.4560 80.7% 203.9s\n",
      "train \t 59: 0.4414 81.5% 203.9s\n",
      "train \t 60: 0.4279 81.8% 203.9s\n",
      "train \t 61: 0.2892 87.8% 203.9s\n",
      "train \t 62: 0.2385 89.9% 203.9s\n",
      "train \t 63: 0.2186 90.7% 203.9s\n",
      "train \t 64: 0.2023 91.3% 203.9s\n",
      "train \t 65: 0.1881 91.8% 203.9s\n",
      "train \t 66: 0.1809 92.2% 203.8s\n",
      "train \t 67: 0.1717 92.6% 203.8s\n",
      "val \t 67: 0.6357 86.0% 4.9s\n",
      "adv FSM \t\t\t 67: 3.2314 54.4% 18.3s\n",
      "adv PGD-20 \t\t\t 67: 4.5555 44.4% 271.8s\n",
      "adv PGD-100 \t\t\t 67: 4.5969 44.0% 1339.5s\n",
      "adv CW-100 \t\t\t 67: 0.9441 44.9% 1333.6s\n",
      "train \t 68: 0.1604 93.2% 203.9s\n",
      "train \t 69: 0.1543 93.4% 203.8s\n",
      "train \t 70: 0.1499 93.5% 203.8s\n",
      "train \t 71: 0.1445 93.9% 203.8s\n",
      "train \t 72: 0.1374 94.1% 203.8s\n",
      "train \t 73: 0.1329 94.3% 204.6s\n",
      "train \t 74: 0.1290 94.6% 203.9s\n",
      "train \t 75: 0.1229 94.9% 203.9s\n",
      "train \t 76: 0.1193 95.0% 204.0s\n",
      "train \t 77: 0.1170 95.1% 204.1s\n",
      "train \t 78: 0.1137 95.3% 204.0s\n",
      "train \t 79: 0.1083 95.4% 203.8s\n",
      "train \t 80: 0.1036 95.7% 203.8s\n",
      "val \t 80: 0.7789 85.4% 4.9s\n",
      "train \t 81: 0.1052 95.6% 203.8s\n",
      "train \t 82: 0.0997 95.9% 203.8s\n",
      "train \t 83: 0.0974 96.0% 203.8s\n",
      "train \t 84: 0.0964 96.1% 203.8s\n",
      "train \t 85: 0.0885 96.4% 203.8s\n",
      "train \t 86: 0.0883 96.3% 204.6s\n",
      "train \t 87: 0.0881 96.4% 203.8s\n",
      "train \t 88: 0.0847 96.5% 203.7s\n",
      "train \t 89: 0.0833 96.6% 203.8s\n",
      "train \t 90: 0.0819 96.6% 203.7s\n",
      "train \t 91: 0.0802 96.7% 203.8s\n",
      "train \t 92: 0.0768 96.9% 203.8s\n",
      "train \t 93: 0.0748 97.0% 204.7s\n",
      "train \t 94: 0.0766 96.9% 203.9s\n",
      "train \t 95: 0.0710 97.1% 203.9s\n",
      "train \t 96: 0.0700 97.1% 203.9s\n",
      "train \t 97: 0.0736 97.1% 203.8s\n",
      "train \t 98: 0.0666 97.3% 204.6s\n",
      "train \t 99: 0.0642 97.5% 203.8s\n",
      "train \t 100: 0.0649 97.4% 203.9s\n",
      "train \t 101: 0.0603 97.6% 203.9s\n",
      "train \t 102: 0.0615 97.5% 203.9s\n",
      "train \t 103: 0.0633 97.4% 203.8s\n",
      "train \t 104: 0.0593 97.6% 203.8s\n",
      "train \t 105: 0.0590 97.7% 203.8s\n",
      "train \t 106: 0.0550 97.9% 203.9s\n",
      "train \t 107: 0.0613 97.6% 203.8s\n",
      "train \t 108: 0.0555 97.8% 203.8s\n",
      "train \t 109: 0.0557 97.8% 203.9s\n",
      "train \t 110: 0.0539 97.9% 204.0s\n",
      "train \t 111: 0.0537 97.8% 204.1s\n",
      "train \t 112: 0.0531 97.9% 203.9s\n",
      "train \t 113: 0.0522 98.0% 203.7s\n",
      "train \t 114: 0.0486 98.1% 203.7s\n",
      "train \t 115: 0.0504 98.1% 203.7s\n",
      "train \t 116: 0.0453 98.2% 203.7s\n",
      "train \t 117: 0.0468 98.2% 203.7s\n",
      "train \t 118: 0.0457 98.3% 203.8s\n",
      "train \t 119: 0.0441 98.3% 203.7s\n",
      "train \t 120: 0.0437 98.4% 203.7s\n",
      "val \t 120: 0.9792 85.4% 4.9s\n",
      "train \t 121: 0.0295 99.0% 203.8s\n",
      "train \t 122: 0.0234 99.2% 203.7s\n",
      "train \t 123: 0.0204 99.3% 203.8s\n",
      "train \t 124: 0.0193 99.3% 203.8s\n",
      "train \t 125: 0.0169 99.5% 203.8s\n",
      "train \t 126: 0.0163 99.4% 204.6s\n",
      "train \t 127: 0.0155 99.5% 203.8s\n",
      "train \t 128: 0.0145 99.5% 203.8s\n",
      "train \t 129: 0.0144 99.5% 203.8s\n",
      "train \t 130: 0.0134 99.6% 203.9s\n",
      "train \t 131: 0.0126 99.6% 203.8s\n",
      "train \t 132: 0.0120 99.6% 203.8s\n",
      "train \t 133: 0.0125 99.6% 203.7s\n",
      "train \t 134: 0.0127 99.6% 203.8s\n",
      "train \t 135: 0.0120 99.6% 203.8s\n",
      "train \t 136: 0.0111 99.7% 203.7s\n",
      "train \t 137: 0.0104 99.7% 203.8s\n",
      "train \t 138: 0.0115 99.7% 203.8s\n",
      "train \t 139: 0.0104 99.7% 204.6s\n",
      "train \t 140: 0.0095 99.7% 203.8s\n",
      "train \t 141: 0.0100 99.7% 203.8s\n",
      "train \t 142: 0.0095 99.7% 203.7s\n",
      "train \t 143: 0.0101 99.6% 203.7s\n",
      "train \t 144: 0.0097 99.7% 203.7s\n",
      "train \t 145: 0.0093 99.7% 203.7s\n",
      "train \t 146: 0.0090 99.7% 203.7s\n",
      "train \t 147: 0.0092 99.7% 203.7s\n",
      "train \t 148: 0.0093 99.7% 203.7s\n",
      "train \t 149: 0.0081 99.7% 203.7s\n",
      "train \t 150: 0.0078 99.8% 203.7s\n",
      "train \t 151: 0.0080 99.7% 203.7s\n",
      "train \t 152: 0.0086 99.7% 203.6s\n",
      "train \t 153: 0.0080 99.8% 203.6s\n",
      "train \t 154: 0.0078 99.8% 203.7s\n",
      "train \t 155: 0.0077 99.7% 203.7s\n",
      "train \t 156: 0.0078 99.7% 203.7s\n",
      "train \t 157: 0.0077 99.8% 203.7s\n",
      "train \t 158: 0.0068 99.8% 203.7s\n",
      "train \t 159: 0.0080 99.7% 203.6s\n",
      "train \t 160: 0.0074 99.7% 203.7s\n",
      "val \t 160: 1.0342 85.4% 4.9s\n",
      "train \t 161: 0.0067 99.8% 203.7s\n",
      "train \t 162: 0.0068 99.8% 203.7s\n",
      "train \t 163: 0.0066 99.8% 203.7s\n",
      "train \t 164: 0.0058 99.8% 203.7s\n",
      "train \t 165: 0.0060 99.8% 203.7s\n",
      "train \t 166: 0.0052 99.9% 203.7s\n",
      "train \t 167: 0.0060 99.8% 203.7s\n",
      "train \t 168: 0.0057 99.8% 203.7s\n",
      "train \t 169: 0.0062 99.8% 203.8s\n",
      "train \t 170: 0.0058 99.8% 203.7s\n",
      "train \t 171: 0.0058 99.8% 203.7s\n",
      "train \t 172: 0.0057 99.8% 203.7s\n",
      "train \t 173: 0.0055 99.8% 203.7s\n",
      "train \t 174: 0.0060 99.8% 203.7s\n",
      "train \t 175: 0.0051 99.9% 203.7s\n",
      "train \t 176: 0.0057 99.8% 203.8s\n",
      "train \t 177: 0.0051 99.9% 203.7s\n",
      "train \t 178: 0.0057 99.8% 203.8s\n",
      "train \t 179: 0.0059 99.8% 203.8s\n",
      "train \t 180: 0.0059 99.8% 203.8s\n",
      "train \t 181: 0.0054 99.9% 203.7s\n",
      "train \t 182: 0.0055 99.8% 203.8s\n",
      "train \t 183: 0.0055 99.8% 203.7s\n",
      "train \t 184: 0.0052 99.9% 204.5s\n",
      "train \t 185: 0.0056 99.8% 203.7s\n",
      "train \t 186: 0.0055 99.8% 203.8s\n",
      "train \t 187: 0.0050 99.9% 203.7s\n",
      "train \t 188: 0.0052 99.9% 203.7s\n",
      "train \t 189: 0.0053 99.8% 203.7s\n",
      "train \t 190: 0.0052 99.8% 203.7s\n",
      "train \t 191: 0.0047 99.9% 203.8s\n",
      "train \t 192: 0.0051 99.9% 203.7s\n",
      "train \t 193: 0.0053 99.8% 203.8s\n",
      "train \t 194: 0.0054 99.8% 203.7s\n",
      "train \t 195: 0.0052 99.8% 203.7s\n",
      "train \t 196: 0.0050 99.8% 203.8s\n",
      "train \t 197: 0.0052 99.9% 203.8s\n",
      "train \t 198: 0.0051 99.8% 203.8s\n",
      "train \t 199: 0.0055 99.8% 203.7s\n",
      "train \t 200: 0.0054 99.8% 203.8s\n",
      "val \t 200: 1.0386 85.5% 4.9s\n",
      "adv FSM \t\t\t 200: 5.8417 54.2% 18.3s\n",
      "adv PGD-20 \t\t\t 200: 8.5961 42.9% 271.7s\n",
      "adv PGD-100 \t\t\t 200: 8.7077 42.5% 1339.0s\n",
      "adv CW-100 \t\t\t 200: 1.3896 42.4% 1336.3s\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "pgd_logs = defaultdict(lambda : defaultdict(lambda : []))\n",
    "for K in PGD_Ks:\n",
    "    print(f'\\n\\n\\n\\n\\ntraining with {K}-PGD------------------------\\n\\n\\n\\n')\n",
    "    model, optimizer, scheduler = build_model(False)\n",
    "    \n",
    "    attack = PGD(K, ϵ, 2.5 * ϵ / K)\n",
    "    \n",
    "    for epoch in range(EPOCHS):  # loop over the dataset multiple times\n",
    "        \n",
    "        \n",
    "        \n",
    "\n",
    "        logs = train_with_replay(1, \n",
    "             model, \n",
    "             trainloader, \n",
    "             optimizer,\n",
    "             epoch,\n",
    "             input_func=lambda inputs, labels: attack(model, inputs, labels))\n",
    "        pgd_logs[K]['train'].append(logs)\n",
    "        \n",
    "        scheduler.step()\n",
    "        if (epoch + 1) % TEST_EVERY == 0:\n",
    "            logs = run_val(model, testloader, epoch)\n",
    "            pgd_logs[K]['test'].append(logs)\n",
    "\n",
    "        if (epoch + 1) == math.ceil(EPOCHS / (K + 1)):\n",
    "            if (epoch + 1) % TEST_EVERY != 0:\n",
    "                logs = run_val(model, testloader, epoch)\n",
    "                pgd_logs[K]['test'].append(logs)\n",
    "            # for K = 1 we want to test at epoch 99 -> True\n",
    "            # for K = 2 we want to test at epoch 67 -> True\n",
    "            \n",
    "            run_attacks(pgd_logs[K], attacks, \n",
    "                attack_names, model, testloader, epoch)\n",
    "            \n",
    "    run_attacks(pgd_logs[K], attacks, \n",
    "                attack_names, model, testloader, epoch)\n",
    "\n",
    "    print('Finished Training')\n",
    "    torch.save(model.state_dict(), f\"snapshots/wresnet-cifar-10-pgk-{K}.pch\")\n",
    "    del model\n",
    "    torch.cuda.empty_cache()\n",
    "    \n",
    "with open('snapshots/pgd_logs.pickle', 'wb') as fd:\n",
    "    pickle.dump(holder_to_dict(pgd_logs), fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Training</th>\n",
       "      <th>Natural Images</th>\n",
       "      <th>FSM</th>\n",
       "      <th>PGD-20</th>\n",
       "      <th>PGD-100</th>\n",
       "      <th>CW-100</th>\n",
       "      <th>Training Time(M)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Natural</td>\n",
       "      <td>$$94.01\\%$$</td>\n",
       "      <td>$$10.77\\%$$</td>\n",
       "      <td>$$0.00\\%$$</td>\n",
       "      <td>$$0.00\\%$$</td>\n",
       "      <td>$$0.00\\%$$</td>\n",
       "      <td>$$235$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Free $m=1$</td>\n",
       "      <td>$$93.60\\%$$</td>\n",
       "      <td>$$15.33\\%$$</td>\n",
       "      <td>$$0.58\\%$$</td>\n",
       "      <td>$$0.54\\%$$</td>\n",
       "      <td>$$0.39\\%$$</td>\n",
       "      <td>$$236$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Free $m=2$</td>\n",
       "      <td>$$89.98\\%$$</td>\n",
       "      <td>$$43.26\\%$$</td>\n",
       "      <td>$$29.28\\%$$</td>\n",
       "      <td>$$29.02\\%$$</td>\n",
       "      <td>$$29.50\\%$$</td>\n",
       "      <td>$$235$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Free $m=4$</td>\n",
       "      <td>$$86.72\\%$$</td>\n",
       "      <td>$$51.03\\%$$</td>\n",
       "      <td>$$41.44\\%$$</td>\n",
       "      <td>$$41.31\\%$$</td>\n",
       "      <td>$$42.19\\%$$</td>\n",
       "      <td>$$235$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Free $m=10$</td>\n",
       "      <td>$$82.49\\%$$</td>\n",
       "      <td>$$52.89\\%$$</td>\n",
       "      <td>$$47.08\\%$$</td>\n",
       "      <td>$$47.02\\%$$</td>\n",
       "      <td>$$46.44\\%$$</td>\n",
       "      <td>$$235$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1-PGD</td>\n",
       "      <td>$$88.54\\%$$</td>\n",
       "      <td>$$97.65\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$459$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2-PGD</td>\n",
       "      <td>$$85.53\\%$$</td>\n",
       "      <td>$$54.24\\%$$</td>\n",
       "      <td>$$42.90\\%$$</td>\n",
       "      <td>$$42.48\\%$$</td>\n",
       "      <td>$$42.43\\%$$</td>\n",
       "      <td>$$680$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1-PGD(EPOCH/K)</td>\n",
       "      <td>$$88.54\\%$$</td>\n",
       "      <td>$$97.74\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$0.01\\%$$</td>\n",
       "      <td>$$230$$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2-PGD(EPOCH/K)</td>\n",
       "      <td>$$85.53\\%$$</td>\n",
       "      <td>$$54.36\\%$$</td>\n",
       "      <td>$$44.38\\%$$</td>\n",
       "      <td>$$44.00\\%$$</td>\n",
       "      <td>$$44.88\\%$$</td>\n",
       "      <td>$$228$$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Training Natural Images          FSM       PGD-20      PGD-100  \\\n",
       "0         Natural    $$94.01\\%$$  $$10.77\\%$$   $$0.00\\%$$   $$0.00\\%$$   \n",
       "1      Free $m=1$    $$93.60\\%$$  $$15.33\\%$$   $$0.58\\%$$   $$0.54\\%$$   \n",
       "2      Free $m=2$    $$89.98\\%$$  $$43.26\\%$$  $$29.28\\%$$  $$29.02\\%$$   \n",
       "3      Free $m=4$    $$86.72\\%$$  $$51.03\\%$$  $$41.44\\%$$  $$41.31\\%$$   \n",
       "4     Free $m=10$    $$82.49\\%$$  $$52.89\\%$$  $$47.08\\%$$  $$47.02\\%$$   \n",
       "5           1-PGD    $$88.54\\%$$  $$97.65\\%$$   $$0.01\\%$$   $$0.01\\%$$   \n",
       "6           2-PGD    $$85.53\\%$$  $$54.24\\%$$  $$42.90\\%$$  $$42.48\\%$$   \n",
       "7  1-PGD(EPOCH/K)    $$88.54\\%$$  $$97.74\\%$$   $$0.01\\%$$   $$0.01\\%$$   \n",
       "8  2-PGD(EPOCH/K)    $$85.53\\%$$  $$54.36\\%$$  $$44.38\\%$$  $$44.00\\%$$   \n",
       "\n",
       "        CW-100 Training Time(M)  \n",
       "0   $$0.00\\%$$          $$235$$  \n",
       "1   $$0.39\\%$$          $$236$$  \n",
       "2  $$29.50\\%$$          $$235$$  \n",
       "3  $$42.19\\%$$          $$235$$  \n",
       "4  $$46.44\\%$$          $$235$$  \n",
       "5   $$0.01\\%$$          $$459$$  \n",
       "6  $$42.43\\%$$          $$680$$  \n",
       "7   $$0.01\\%$$          $$230$$  \n",
       "8  $$44.88\\%$$          $$228$$  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fmt = lambda x: f'$${x * 100:.2f}\\%$$'\n",
    "d = {}\n",
    "def get_good_log(x, K):\n",
    "    return sorted(x, key=lambda j: abs((j.epoch + 1) - (EPOCHS // (K+1))))[-1]\n",
    "d['Training'] = ['Natural', \n",
    "         *[f'Free $m={K}$' for K in free_Ks],\n",
    "         *[f'{K}-PGD' for K in PGD_Ks],\n",
    "         *[f'{K}-PGD(EPOCH/K)' for K in PGD_Ks]]\n",
    "\n",
    "x = [srl[1]['test'][-1].acc,\n",
    "         *[free_logs[K]['test'][-1].acc for K in free_Ks],\n",
    "         *[pgd_logs[K]['test'][-1].acc for K in PGD_Ks],\n",
    "        *[get_good_log(pgd_logs[K]['test'], K).acc for K in PGD_Ks]]\n",
    "\n",
    "d['Natural Images'] = list(map(fmt, x))\n",
    "            \n",
    "for name in attack_names:\n",
    "    n = f'adv_test/{name}'\n",
    "    \n",
    "    x = [srl[1][n][-1].acc]\n",
    "    \n",
    "    for K in free_Ks:\n",
    "        x.append(free_logs[K][n][-1].acc)\n",
    "    \n",
    "    for K in PGD_Ks:\n",
    "        x.append(pgd_logs[K][n][-1].acc)\n",
    "    for K in PGD_Ks:\n",
    "        x.append(pgd_logs[K][n][-2].acc)\n",
    "    d[name] = list(map(fmt, x))\n",
    "        \n",
    "tt = lambda x: sum(i.time for i in x)\n",
    "fmt = lambda x: f'$${math.ceil(x / 60)}$$'\n",
    "x = [srl[1]['train'],\n",
    "    *[free_logs[K]['train'] for K in free_Ks],\n",
    "    *[pgd_logs[K]['train'] for K in PGD_Ks],\n",
    "    *[pgd_logs[K]['train'][:math.ceil(EPOCHS / (K + 1))] for K in PGD_Ks]]\n",
    "\n",
    "d['Training Time(M)'] = list(map(lambda x: fmt(tt(x)), x))\n",
    "\n",
    "df = pd.DataFrame(d)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('figures/grid.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_latex('figures/grid.tex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for K in training_with_replay_Ks[1:]:\n",
    "    print(f'\\n\\n\\n\\n\\ntraining with {K} replays------------------------\\n\\n\\n\\n')\n",
    "\n",
    "    model, optimizer, scheduler = build_model(False, K=K)\n",
    "        \n",
    "    for epoch in range(int(EPOCHS / K)): # loop over the dataset multiple times\n",
    "            \n",
    "        logs = train_with_replay(K, model, trainloader, optimizer, epoch)\n",
    "        \n",
    "        scheduler.step()\n",
    "        srl[K]['train'].append(logs)\n",
    "        if (epoch * K + K) % TEST_EVERY == 0:\n",
    "            # valdiation loss\n",
    "            logs = run_val(model, testloader, epoch)\n",
    "            srl[K]['test'].append(logs)\n",
    "    run_attacks(srl[K], attacks, \n",
    "                attack_names, model, testloader, epoch)\n",
    "    print('Finished Training')\n",
    "    torch.save(model.state_dict(), f\"wresnet-cifar-10-normal-{K}.pch\")\n",
    "    del model\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "with open('snapshots/srl.pickle', 'wb') as fd:\n",
    "    pickle.dump(holder_to_dict(srl), fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax2, ax1) = plt.subplots(ncols=2, figsize=(15,7))\n",
    "\n",
    "y = [srl[K][\"test\"][-1].acc * 100 for K in training_with_replay_Ks]\n",
    "bars = ax1.bar([f'$m={K}$' for K in training_with_replay_Ks], y)\n",
    "for (i, bar) in zip(y, bars):\n",
    "    t = ax1.text(bar.get_x() + bar.get_width() /2 - 0.07 , bar.get_height() + 0.10, f'{i:0.1f}%')\n",
    "for ax in [ax1, ax2]:\n",
    "    ax.set_xlabel('number of replay steps $m$')\n",
    "ax1.set_ylabel('validation accuracy ($\\%$)')\n",
    "\n",
    "ax2.set_ylabel('validation loss (KL)')\n",
    "y = [srl[K][\"test\"][-1].loss for K in training_with_replay_Ks]\n",
    "bars = ax2.bar([f'$m={K}$' for K in training_with_replay_Ks], y)\n",
    "for (i, bar) in zip(y, bars):\n",
    "    t = ax2.text(bar.get_x() + bar.get_width() /2 - 0.07 , bar.get_height() + 0.10, f'{i:0.1f}')\n",
    "def savefig(fig, name, f=['svg', 'pdf', 'png']):\n",
    "    for e in f:\n",
    "        fig.savefig('figures/' + name + '.' + e)\n",
    "savefig(fig, 'cost_of_replay')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}