get_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## This code calculate statistic indicators including AUC, SEN, SPE, PPV, NPV, ACC of ten folds of training, validation and test sets. It also generates each fold's ROC curve with the best threshold."
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "import time\r\n",
    "import re\r\n",
    "import os\r\n",
    "from plotly import graph_objs as go\r\n",
    "from sklearn.metrics import roc_auc_score\r\n",
    "from scipy.stats import sem\r\n",
    "from sklearn import svm\r\n",
    "import pandas as pd\r\n",
    "import math\r\n",
    "import shutil\r\n",
    "import random\r\n",
    "from collections import namedtuple\r\n",
    "import copy\r\n",
    "import numpy as np\r\n",
    "import torch\r\n",
    "import torch.nn as nn\r\n",
    "import torch.nn.functional as F\r\n",
    "import torch.optim as optim\r\n",
    "import torch.optim.lr_scheduler as lr_scheduler\r\n",
    "from torch.optim.lr_scheduler import _LRScheduler\r\n",
    "import torch.utils.data as data\r\n",
    "import torchvision.transforms as transforms\r\n",
    "import torchvision.datasets as datasets\r\n",
    "import torchvision.models as models\r\n",
    "from torch.utils.data import Dataset, DataLoader\r\n",
    "from torchvision import utils\r\n",
    "\r\n",
    "from sklearn import decomposition\r\n",
    "from sklearn import manifold\r\n",
    "from sklearn import metrics\r\n",
    "from sklearn import model_selection\r\n",
    "from sklearn.metrics import confusion_matrix\r\n",
    "\r\n",
    "import matplotlib.pyplot as plt\r\n",
    "%matplotlib inline\r\n",
    "\r\n",
    "\r\n",
    "# import roc\r\n",
    "\r\n",
    "# set seed to make sure the results are reproducible\r\n",
    "SEED = 123\r\n",
    "random.seed(SEED)\r\n",
    "np.random.seed(SEED)\r\n",
    "torch.manual_seed(SEED)\r\n",
    "torch.cuda.manual_seed(SEED)\r\n",
    "torch.backends.cudnn.deterministic = True\r\n",
    "\r\n",
    "# change directory of data\r\n",
    "datapath = os.path.join(\".data\", \"hcb\", \"cv1\")\r\n",
    "train_dir = os.path.join(datapath, 'train')\r\n",
    "val_dir = os.path.join(datapath, 'val')\r\n",
    "test_dir = os.path.join(datapath, 'test')\r\n",
    "\r\n",
    "pretrained_size = (224, 224)\r\n",
    "pretrained_means = [0.485, 0.456, 0.406]\r\n",
    "pretrained_stds = [0.229, 0.224, 0.225]\r\n",
    "\r\n",
    "train_transforms = transforms.Compose([\r\n",
    "    transforms.Resize(pretrained_size),\r\n",
    "    #    transforms.ColorJitter(brightness = 1, contrast = 0.5),\r\n",
    "    # transforms.RandomHorizontalFlip(0.2),\r\n",
    "    # transforms.RandomVerticalFlip(0.2),\r\n",
    "    # transforms.RandomCrop(pretrained_size, padding = 10),\r\n",
    "    # transforms.CenterCrop(pretrained_size),\r\n",
    "    # transforms.ColorJitter(brightness = 1, contrast = 0.5),\r\n",
    "    transforms.ToTensor(),\r\n",
    "    transforms.Normalize(mean=pretrained_means,\r\n",
    "                         std=pretrained_stds)\r\n",
    "])\r\n",
    "\r\n",
    "test_transforms = transforms.Compose([\r\n",
    "    transforms.Resize(pretrained_size),\r\n",
    "    transforms.ToTensor(),\r\n",
    "    transforms.Normalize(mean=pretrained_means,\r\n",
    "                         std=pretrained_stds)\r\n",
    "])\r\n",
    "# inorder to load img with it's label\r\n",
    "\r\n",
    "\r\n",
    "class MyImageFolder(datasets.ImageFolder):\r\n",
    "    def __getitem__(self, index):\r\n",
    "        path, _ = self.imgs[index]  # img path, label\r\n",
    "        # return image path\r\n",
    "        return super(MyImageFolder, self).__getitem__(index), path\r\n",
    "\r\n",
    "\r\n",
    "BATCH_SIZE = 64  # 4\r\n",
    "# load data from data directory\r\n",
    "train_data = MyImageFolder(root=train_dir,\r\n",
    "                           transform=train_transforms)\r\n",
    "test_data = MyImageFolder(root=test_dir,\r\n",
    "                          transform=test_transforms)\r\n",
    "valid_data = MyImageFolder(root=val_dir,\r\n",
    "                           transform=test_transforms)\r\n",
    "train_iterator = data.DataLoader(train_data,\r\n",
    "                                 shuffle=True, drop_last=True,\r\n",
    "                                 batch_size=BATCH_SIZE)\r\n",
    "valid_iterator = data.DataLoader(valid_data, drop_last=True,\r\n",
    "                                 batch_size=BATCH_SIZE)\r\n",
    "test_iterator = data.DataLoader(test_data, drop_last=True,\r\n",
    "                                batch_size=BATCH_SIZE)\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Basic Model"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "model = torch.hub.load('pytorch/vision:v0.6.0', 'resnet50', pretrained=True)\r\n",
    "# change output dimension to what we need\r\n",
    "IN_FEATURES = model.fc.in_features\r\n",
    "OUTPUT_DIM = 2\r\n",
    "model.fc = nn.Linear(IN_FEATURES, OUTPUT_DIM)\r\n",
    "fc1 = nn.Linear(IN_FEATURES, 32)\r\n",
    "fc2 = nn.Linear(32, OUTPUT_DIM)\r\n",
    "model.fc = nn.Sequential(fc1, fc2)\r\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\r\n",
    "model = model.to(device)\r\n",
    "model.load_state_dict(torch.load('basic-model-cv1.pt',\r\n",
    "                                 map_location=torch.device('cpu')))\r\n",
    "\r\n",
    "# criterion = nn.CrossEntropyLoss()\r\n",
    "criterion = nn.CrossEntropyLoss(weight=torch.tensor([1/256, 1/410]).to(device))\r\n",
    "criterion = criterion.to(device)\r\n",
    "\r\n",
    "FOUND_LR = 1e-4\r\n",
    "weight_decay = 1e-4\r\n",
    "params = [\r\n",
    "    {'params': model.conv1.parameters(), 'lr': FOUND_LR / 10},\r\n",
    "    {'params': model.bn1.parameters(), 'lr': FOUND_LR / 10},\r\n",
    "    {'params': model.layer1.parameters(), 'lr': FOUND_LR / 8},\r\n",
    "    {'params': model.layer2.parameters(), 'lr': FOUND_LR / 6},\r\n",
    "    {'params': model.layer3.parameters(), 'lr': FOUND_LR / 4},\r\n",
    "    {'params': model.layer4.parameters(), 'lr': FOUND_LR / 2},\r\n",
    "    {'params': model.fc.parameters(), 'lr': FOUND_LR / 2}\r\n",
    "]\r\n",
    "optimizer = optim.Adam(params, lr=FOUND_LR, weight_decay=weight_decay)\r\n",
    "# scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)\r\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(\r\n",
    "    optimizer, 'min', factor=0.8, patience=5, min_lr=1e-9)\r\n"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "Using cache found in C:\\Users\\Admin/.cache\\torch\\hub\\pytorch_vision_v0.6.0\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "# some functions for modelling\r\n",
    "def get_predictions(model, iterator):\r\n",
    "    model.eval()\r\n",
    "    images = []\r\n",
    "    labels = []\r\n",
    "    probs = []\r\n",
    "    top_pred = []\r\n",
    "    with torch.no_grad():\r\n",
    "        for (xy, _) in iterator:\r\n",
    "            x, y = xy\r\n",
    "            y_pred = model(x)\r\n",
    "            y_prob = F.softmax(y_pred, dim=-1)\r\n",
    "            _, _top_pred = torch.max(y_pred, 1)\r\n",
    "            images.append(x.cpu())\r\n",
    "            labels.append(y.cpu())\r\n",
    "            probs.append(y_prob.cpu())\r\n",
    "            top_pred.append(_top_pred.cpu())\r\n",
    "    images = torch.cat(images, dim=0)\r\n",
    "    labels = torch.cat(labels, dim=0)\r\n",
    "    probs = torch.cat(probs, dim=0)\r\n",
    "    top_pred = torch.cat(top_pred, dim=0)\r\n",
    "    return images, labels, probs, top_pred\r\n",
    "\r\n",
    "model.to(\"cpu\")\r\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "\r\n",
    "def roc_results(y_true, y_pred, path):\r\n",
    "    r\"\"\"\r\n",
    "    Return: sensitivity, specificity, ppv, npv, accuracy, AUC, threshold\r\n",
    "    this function is used to calculate AUC,sensitivity,specificity,ppv,npv,accuracy\r\n",
    "    baesed on each threshold and plot the ROC curve. It will also choose the best threshold, which means that this threshold lead to the highest accuracy.\r\n",
    "    input: true, pred, path\r\n",
    "    \"\"\"\r\n",
    "# calculate the target vector from roc\r\n",
    "    fpr, sen, threshold = metrics.roc_curve(y_true, y_pred)\r\n",
    "    spe = 1-fpr\r\n",
    "    pos_num = np.sum(y_true == 1)\r\n",
    "    neg_num = len(y_true)-pos_num\r\n",
    "    tp = sen*pos_num\r\n",
    "    tn = spe*neg_num\r\n",
    "    fn = pos_num-tp\r\n",
    "    fp = neg_num-tn\r\n",
    "    ppv = tp/(tp+fp+1e-16)\r\n",
    "    npv = tn/(tn+fn+1e-16)\r\n",
    "    acc = (tp+tn)/len(y_true)\r\n",
    "# the best point data\r\n",
    "    j_statistic = sen+spe-1  # (based on J_statistic)\r\n",
    "    ind = np.argmax(j_statistic)+0\r\n",
    "    pred_p = tp[ind]+fp[ind]\r\n",
    "    pred_n = tn[ind]+fn[ind]\r\n",
    "    best_acc = acc[ind]\r\n",
    "    best_sen = sen[ind]\r\n",
    "    best_spe = spe[ind]\r\n",
    "    best_ppv = tp[ind]/pred_p\r\n",
    "    best_npv = tn[ind]/pred_n\r\n",
    "    AUC = metrics.auc(fpr, sen)\r\n",
    "# plot\r\n",
    "    plt.plot(fpr, sen)\r\n",
    "    plt.plot(np.linspace(0, 1, 10), np.linspace(0, 1, 10))\r\n",
    "    plt.plot(fpr[ind], sen[ind], 'r.', markersize=9)\r\n",
    "    text = \"AUC={:.2f}\\nSEN={:.2f}\\nSPE={:.2f}\\nPPV={:.2f}\\nNPV={:.2f}\\nACC={:.2f}\".format(\r\n",
    "        AUC, sen[ind], spe[ind], best_ppv, best_npv, best_acc)\r\n",
    "    plt.text(0.82, 0.1, text, fontsize=10,\r\n",
    "             style=\"italic\", horizontalalignment=\"center\")\r\n",
    "    plt.xlabel('1-Specificity')\r\n",
    "    plt.ylabel('Sensitivity')\r\n",
    "    plt.title('ROC Curve')\r\n",
    "    path = path+\".jpg\"\r\n",
    "    plt.savefig(path)\r\n",
    "    plt.show()\r\n",
    "    return sen, spe, ppv, npv, acc, AUC, threshold"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "#test cv\r\n",
    "for ii in range(1, 11):\r\n",
    "    i = str(ii)\r\n",
    "    datapath = os.path.join(\".data\", \"hcb\", \"cv\"+i)\r\n",
    "    test_dir = os.path.join(datapath, 'test')\r\n",
    "    test_data = MyImageFolder(root=test_dir,\r\n",
    "                              transform=test_transforms)\r\n",
    "    test_iterator = data.DataLoader(test_data, drop_last=True,\r\n",
    "                                    batch_size=BATCH_SIZE)\r\n",
    "    model.load_state_dict(torch.load(\r\n",
    "        'basic-model-cv'+i+'.pt', map_location=torch.device('cpu')))\r\n",
    "    cv = i\r\n",
    "    path = 'getpred/testcv'+cv\r\n",
    "    _, labels, probs, _ = get_predictions(model, test_iterator)\r\n",
    "    pred = probs[:, 0]\r\n",
    "    sen, spe, ppv, npv, acc, auc, _ = roc_results(\r\n",
    "        1-labels.numpy(), pred.numpy(), path)"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "RuntimeError",
     "evalue": "Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-eae8dea436e0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mcv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0mpath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'getpred/testcv'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcv\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m     \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_predictions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_iterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m     \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprobs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m     sen, spe, ppv, npv, acc, auc, _ = roc_results(\n",
      "\u001b[1;32m<ipython-input-3-d533e0c804e9>\u001b[0m in \u001b[0;36mget_predictions\u001b[1;34m(model, iterator)\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mxy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m             \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m             \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m             \u001b[0my_prob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mF\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m             \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_top_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    726\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 727\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[0;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torchvision\\models\\resnet.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    219\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 220\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    221\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    222\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torchvision\\models\\resnet.py\u001b[0m in \u001b[0;36m_forward_impl\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    201\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_forward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m         \u001b[1;31m# See note [TorchScript super()]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 203\u001b[1;33m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    204\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    726\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 727\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[0;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torch\\nn\\modules\\conv.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    422\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 423\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_conv_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    424\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    425\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\pytorch\\lib\\site-packages\\torch\\nn\\modules\\conv.py\u001b[0m in \u001b[0;36m_conv_forward\u001b[1;34m(self, input, weight)\u001b[0m\n\u001b[0;32m    418\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[0;32m    419\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[1;32m--> 420\u001b[1;33m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[0;32m    421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    422\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#train cv\r\n",
    "for ii in range(1, 11):\r\n",
    "    i = str(ii)\r\n",
    "    # change directory of data\r\n",
    "    datapath = os.path.join(\".data\", \"hcb\", \"cv\"+i)\r\n",
    "    train_dir = os.path.join(datapath, 'train')\r\n",
    "    train_data = MyImageFolder(root=train_dir,\r\n",
    "                               transform=train_transforms)\r\n",
    "    train_iterator = data.DataLoader(train_data,\r\n",
    "                                     shuffle=True, drop_last=True,\r\n",
    "                                     batch_size=BATCH_SIZE)\r\n",
    "    model.load_state_dict(torch.load(\r\n",
    "        'basic-model-cv'+i+'.pt', map_location=torch.device('cpu')))\r\n",
    "    cv = i\r\n",
    "    path = 'getpred/traincv'+cv\r\n",
    "    _, labels, probs, _ = get_predictions(model, train_iterator)\r\n",
    "    pred = probs[:, 0]\r\n",
    "    sen, spe, ppv, npv, acc, auc, _ = roc_results(\r\n",
    "        1-labels.numpy(), pred.numpy(), path)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "#validation cv\r\n",
    "for ii in range(1, 11):\r\n",
    "    i = str(ii)\r\n",
    "    # change directory of data\r\n",
    "    datapath = os.path.join(\".data\", \"hcb\", \"cv\"+i)\r\n",
    "    val_dir = os.path.join(datapath, 'val')\r\n",
    "    valid_data = MyImageFolder(root=val_dir,\r\n",
    "                               transform=test_transforms)\r\n",
    "    valid_iterator = data.DataLoader(valid_data, drop_last=True,\r\n",
    "                                     batch_size=BATCH_SIZE)\r\n",
    "    model.load_state_dict(torch.load(\r\n",
    "        'basic-model-cv'+i+'.pt', map_location=torch.device('cpu')))\r\n",
    "    cv = i\r\n",
    "    path = 'getpred/valcv'+cv\r\n",
    "    _, labels, probs, _ = get_predictions(model, valid_iterator)\r\n",
    "    pred = probs[:, 0]\r\n",
    "    sen, spe, ppv, npv, acc, auc, _ = roc_results(\r\n",
    "        1-labels.numpy(), pred.numpy(), path)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.6.7",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.7 64-bit ('pytorch': conda)"
  },
  "interpreter": {
   "hash": "62eb3f0dc0d696e2dd85d645fd9023f02c025c88b4bd5b0bd938b54b93374391"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}