004_k_fold_cross_validation.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.10 64-bit ('ma': conda)",
   "metadata": {
    "interpreter": {
     "hash": "739f6139ea16146f6825468ed5e82eb0c1c232f377b4e45bfd13eaa0a4a5ceb5"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# k-Fold Cross Validation\n",
    "https://www.machinecurve.com/index.php/2021/02/03/how-to-use-k-fold-cross-validation-with-pytorch/"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# system imports\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "# additional imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import torch\n",
    "\n",
    "# internal imports\n",
    "from utils import CoughNet\n",
    "\n",
    "# device config\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "source": [
    "## Hyperparameters"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "hparams = {    \n",
    "    'dataset': 'data/prepared_data_balanced.csv',\n",
    "    'epochs': 20,\n",
    "    'batch_size': 16,\n",
    "    'lr': 1e-3,\n",
    "    'features': [\n",
    "        'chroma_stft', 'rmse', 'spectral_centroid', 'spectral_bandwidth', 'rolloff', 'zero_crossing_rate',\n",
    "        'mfcc1', 'mfcc2', 'mfcc3', 'mfcc4', 'mfcc5', 'mfcc6', 'mfcc7', 'mfcc8', 'mfcc9', 'mfcc10', \n",
    "        'mfcc11', 'mfcc12', 'mfcc13', 'mfcc14', 'mfcc15', 'mfcc16', 'mfcc17', 'mfcc18', 'mfcc19', 'mfcc20'\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "source": [
    "## Prepare Data"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_features = pd.read_csv(hparams['dataset'])\n",
    "X = np.array(df_features[hparams['features']], dtype=np.float32)\n",
    "\n",
    "encoder = LabelEncoder()\n",
    "y = encoder.fit_transform(df_features['label'])"
   ]
  },
  {
   "source": [
    "## K-fold Cross Validation model evaluation"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "K-FOLD CROSS VALIDATION RESULTS FOR 8 FOLDS\n",
      "--------------------------------------------\n",
      "|         | Train Accuracy | Test Accuracy |\n",
      "--------------------------------------------\n",
      "| Fold 0  |       100.00 % |       94.12 % |\n",
      "| Fold 1  |       100.00 % |       88.24 % |\n",
      "| Fold 2  |       100.00 % |       94.12 % |\n",
      "| Fold 3  |       100.00 % |       76.47 % |\n",
      "| Fold 4  |       100.00 % |       88.24 % |\n",
      "| Fold 5  |       100.00 % |       94.12 % |\n",
      "| Fold 6  |       100.00 % |       100.00 % |\n",
      "| Fold 7  |       96.61 % |       87.50 % |\n",
      "--------------------------------------------\n",
      "| Average |       99.58 % |       90.35 % |\n"
     ]
    }
   ],
   "source": [
    "k_folds = 8\n",
    "kfold = KFold(n_splits=k_folds, shuffle=True, random_state=42)\n",
    "indices = np.arange(len(y))\n",
    "results_train = []\n",
    "results_test = []\n",
    "\n",
    "def train(loader_train, model, optimizer, epoch):\n",
    "    model.train()\n",
    "    running_correct = 0.0\n",
    "    total = 0\n",
    "    for batch_ndx, sample in enumerate(loader_train): \n",
    "        features, labels = sample[0].to(device), sample[1].to(device) \n",
    "\n",
    "        # forward pass and loss calculation\n",
    "        outputs = model(features)\n",
    "        loss = criterion(outputs, labels)  \n",
    "        \n",
    "        # backward pass    \n",
    "        loss.backward()\n",
    "        \n",
    "        # update weights\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # calculate metrics\n",
    "        predictions = torch.argmax(outputs.data, 1)\n",
    "        running_correct += (predictions == labels).sum().item()\n",
    "        total += labels.shape[0]\n",
    "\n",
    "    return running_correct / total\n",
    "\n",
    "def evaluate(loader_test, model, epoch):\n",
    "    model.eval()\n",
    "    running_correct = 0.0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_ndx, sample in enumerate(loader_test):\n",
    "            features, labels = sample[0].to(device), sample[1].to(device) \n",
    "\n",
    "            # forward pass and loss calculation\n",
    "            outputs = model(features)\n",
    "            loss = criterion(outputs, labels)  \n",
    "\n",
    "            # calculate metrics\n",
    "            predictions = torch.argmax(outputs.data, 1)\n",
    "            running_correct += (predictions == labels).sum().item()\n",
    "            total += labels.shape[0]\n",
    "\n",
    "    return running_correct / total\n",
    "\n",
    "print(f'K-FOLD CROSS VALIDATION RESULTS FOR {k_folds} FOLDS')\n",
    "print('--------------------------------------------')\n",
    "print('|         | Train Accuracy | Test Accuracy |')\n",
    "print('--------------------------------------------')\n",
    "\n",
    "for fold, (train_ids, test_ids) in enumerate(kfold.split(indices)):\n",
    "    X_train = X[train_ids]\n",
    "    y_train = y[train_ids]\n",
    "    X_test = X[test_ids]\n",
    "    y_test = y[test_ids]\n",
    "    \n",
    "    # scale data\n",
    "    scaler = StandardScaler()\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    \n",
    "    # create pytorch dataloader\n",
    "    torch.manual_seed(42)\n",
    "    train_dataset = torch.utils.data.TensorDataset(torch.Tensor(X_train), torch.Tensor(y_train).long())\n",
    "    test_dataset = torch.utils.data.TensorDataset(torch.Tensor(X_test), torch.Tensor(y_test).long())\n",
    "    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=hparams['batch_size'], shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=hparams['batch_size'], shuffle=False)\n",
    "    \n",
    "    # create model\n",
    "    model = CoughNet(len(hparams['features'])).to(device)\n",
    "\n",
    "    # Construct loss and optimizer\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=hparams['lr'])\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "    # training loop\n",
    "    for epoch in range(hparams['epochs']):\n",
    "        train_accuracy = train(train_loader, model, optimizer, epoch)\n",
    "        eval_accuracy = evaluate(test_loader, model, epoch)\n",
    "    results_train.append(train_accuracy) \n",
    "    results_test.append(eval_accuracy) \n",
    "    print(f'| Fold {fold}  |       {train_accuracy*100:.2f} % |       {eval_accuracy*100:.2f} % |')\n",
    "\n",
    "print('--------------------------------------------')\n",
    "print(f'| Average |       {np.mean(results_train)*100:.2f} % |       {np.mean(results_test)*100:.2f} % |')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}