04_Model_Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {
    "vscode": {
     "languageId": "raw"
    }
   },
   "source": [
    "---\n",
    "title: \"ClassifyAnything Part 4: Model Evaluation\"\n",
    "author: \"Joseph Chao-Chung Kuo (ckuo@ukaachen.de)\"\n",
    "date: today\n",
    "output:\n",
    "  general:\n",
    "    output_stdout: true\n",
    "    output_error: false\n",
    "    output: true\n",
    "format:\n",
    "  html:\n",
    "    toc: true\n",
    "    toc-depth: 3\n",
    "    html-math-method: katex\n",
    "    code-fold: true\n",
    "    code-tools: true\n",
    "    number_sections: true\n",
    "    embed-resources: true\n",
    "    page-layout: full\n",
    "execute:\n",
    "    echo: true\n",
    "    warning: false\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Supervised learning for classification_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "import plotly.graph_objects as go\n",
    "from sklearn.metrics import (accuracy_score, \n",
    "                             precision_score, \n",
    "                             recall_score, \n",
    "                             f1_score, \n",
    "                             confusion_matrix,\n",
    "                             roc_curve,\n",
    "                             auc)\n",
    "import joblib\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "# import ace_tools as tools\n",
    "import warnings\n",
    "warnings.filterwarnings('always')  # \"error\", \"ignore\", \"always\", \"default\", \"module\" or \"once\"\n",
    "\n",
    "with open(\"outputs/03_Variables.pkl\", 'rb') as file:\n",
    "    (X_train, X_test, y_train, y_test, kfold) = pickle.load(file)\n",
    "\n",
    "model_logistic_regression = joblib.load('outputs/03-1_Logistic_Regression_final_model.joblib')\n",
    "model_decision_tree = joblib.load('outputs/03-2_Decision_Tree_final_model.joblib')\n",
    "model_random_forest = joblib.load('outputs/03-3_Random_Forest_final_model.joblib')\n",
    "model_SVM = joblib.load('outputs/03-4_SVM_final_model.joblib')\n",
    "\n",
    "trained_models = {\"Logistic Regression\": model_logistic_regression,\n",
    "                  \"Decision Tree\": model_decision_tree,\n",
    "                  \"Random Forest\": model_random_forest,\n",
    "                  \"SVM\": model_SVM}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Benchmark the trained models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create lists to store evaluation metrics for each model\n",
    "model_names = []\n",
    "accuracies = []\n",
    "precisions = []\n",
    "recalls = []\n",
    "f1_scores = []\n",
    "conf_matrices = []\n",
    "misclassified_samples = {}\n",
    "\n",
    "# Loop through each trained model\n",
    "for model_name in trained_models:\n",
    "    # Make predictions on the test dataset\n",
    "    y_pred = trained_models[model_name].predict(X_test)\n",
    "    \n",
    "    # Calculate evaluation metrics\n",
    "    accuracy = accuracy_score(y_test, y_pred)\n",
    "    precision = precision_score(y_test, y_pred, average='weighted')\n",
    "    recall = recall_score(y_test, y_pred, average='weighted')\n",
    "    f1 = f1_score(y_test, y_pred, average='weighted')\n",
    "    conf_matrix = confusion_matrix(y_test, y_pred)\n",
    "    \n",
    "    # Append metrics to lists\n",
    "    model_names.append(model_name)\n",
    "    accuracies.append(accuracy)\n",
    "    precisions.append(precision)\n",
    "    recalls.append(recall)\n",
    "    f1_scores.append(f1)\n",
    "    conf_matrices.append(conf_matrix)\n",
    "    \n",
    "    # Find indices of misclassified samples\n",
    "    misclassified_indices = [i for i, (true, pred) in enumerate(zip(y_test, y_pred)) if true != pred]\n",
    "    \n",
    "    # Store misclassified samples with their true and predicted labels\n",
    "    misclassified_samples[model_name] = {\n",
    "        'sample_names': X_test.index[misclassified_indices],  # Assuming X_test is a DataFrame\n",
    "        'true_label': np.array(y_test)[misclassified_indices],\n",
    "        'predicted_label': y_pred[misclassified_indices]\n",
    "    }\n",
    "\n",
    "# Create a DataFrame to store the evaluation results\n",
    "evaluation_df = pd.DataFrame({\n",
    "    'Model': model_names,\n",
    "    'Accuracy': accuracies,\n",
    "    'Precision': precisions,\n",
    "    'Recall': recalls,\n",
    "    'F1 Score': f1_scores,\n",
    "    'Confusion Matrix': conf_matrices\n",
    "})\n",
    "\n",
    "# Display the evaluation results\n",
    "display(evaluation_df.drop(columns=\"Confusion Matrix\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the evaluation metrics bar chart using Plotly\n",
    "fig = go.Figure()\n",
    "\n",
    "fig.add_trace(go.Bar(x=model_names, y=accuracies, name='Accuracy'))\n",
    "fig.add_trace(go.Bar(x=model_names, y=f1_scores, name='F1 Score'))\n",
    "fig.add_trace(go.Bar(x=model_names, y=precisions, name='Precision'))\n",
    "fig.add_trace(go.Bar(x=model_names, y=recalls, name='Recall'))\n",
    "\n",
    "fig.update_layout(title='Model Evaluation Metrics',\n",
    "                  xaxis_title='Model',\n",
    "                  yaxis_title='Score',\n",
    "                  barmode='group')\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### List of mislabeled samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "misclassified_df = pd.DataFrame(columns=['Model', 'Sample Name', 'True Label', 'Predicted Label'])\n",
    "\n",
    "# Loop through each model and collect misclassified sample information\n",
    "for model_name in misclassified_samples:\n",
    "    misclassified = misclassified_samples[model_name]\n",
    "    \n",
    "    # Create a temporary DataFrame for the current model's misclassified samples\n",
    "    temp_df = pd.DataFrame({\n",
    "        'Model': model_name,  # Add model name to each row\n",
    "        'Sample Name': misclassified['sample_names'],\n",
    "        'True Label': misclassified['true_label'],\n",
    "        'Predicted Label': misclassified['predicted_label']\n",
    "    })\n",
    "    \n",
    "    # Append to the main DataFrame\n",
    "    misclassified_df = pd.concat([misclassified_df, temp_df], ignore_index=True)\n",
    "\n",
    "# Display the resulting DataFrame with misclassified samples\n",
    "display(misclassified_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 Confusion matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assume class_labels and conf_matrices are defined\n",
    "class_labels = np.unique(y_train)\n",
    "\n",
    "# Create a subplot grid to plot multiple confusion matrices (2 rows for aesthetics)\n",
    "n_models = len(trained_models)\n",
    "fig, axes = plt.subplots(nrows=4, ncols=1, figsize=(6, 13))\n",
    "\n",
    "# Flatten the axes array to easily iterate (since we may have fewer models than subplot slots)\n",
    "axes = axes.flatten()\n",
    "\n",
    "# Plot each confusion matrix using a heatmap\n",
    "for i, model_name in enumerate(trained_models):\n",
    "    sns.heatmap(conf_matrices[i], annot=True, cmap='Blues', fmt='g', ax=axes[i])\n",
    "    axes[i].set_title(model_name)\n",
    "    axes[i].set_xlabel('Predicted')\n",
    "    axes[i].set_ylabel('True')\n",
    "    \n",
    "    # Set the correct number of ticks based on the number of classes\n",
    "    axes[i].set_xticks(np.arange(len(class_labels)))\n",
    "    axes[i].set_yticks(np.arange(len(class_labels)))\n",
    "    \n",
    "    # Set the class labels as tick labels\n",
    "    axes[i].set_xticklabels(class_labels, rotation=45, ha='right', fontsize=10)\n",
    "    axes[i].set_yticklabels(class_labels, rotation=0, fontsize=10)\n",
    "\n",
    "# Remove unused subplots if necessary\n",
    "for j in range(i + 1, len(axes)):\n",
    "    fig.delaxes(axes[j])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%ignore\n",
    "# Create the ROC curve plot using Plotly\n",
    "# For single class\n",
    "# fig = go.Figure()\n",
    "\n",
    "# for model_name in trained_models:\n",
    "#     trained_models[model_name].fit(X_train, y_train)\n",
    "#     y_pred_prob = trained_models[model_name].predict_proba(X_test)[:, 1]\n",
    "#     fpr, tpr, _ = roc_curve(y_test, y_pred_prob)\n",
    "#     roc_auc = auc(fpr, tpr)\n",
    "    \n",
    "#     # Add ROC curve trace for each classifier\n",
    "#     fig.add_trace(go.Scatter(x=fpr, y=tpr, mode='lines', name=f'{model_name} (AUC = {roc_auc:.2f})'))\n",
    "\n",
    "# # Add diagonal line representing random guessing\n",
    "# fig.add_trace(go.Scatter(x=[0, 1], y=[0, 1], mode='lines', line=dict(color='gray', dash='dash'), name='Random Guessing'))\n",
    "\n",
    "# fig.update_layout(title='ROC Curve Comparison for Different Classifiers',\n",
    "#                   xaxis_title='False Positive Rate',\n",
    "#                   yaxis_title='True Positive Rate',\n",
    "#                   legend_title='Classifiers',\n",
    "#                   showlegend=True)\n",
    "\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the ROC curve plot using Plotly\n",
    "# For multiple classes\n",
    "for model_name in trained_models:\n",
    "    fig = go.Figure()\n",
    "    trained_models[model_name].fit(X_train, y_train)\n",
    "    # Get the probabilities for each class\n",
    "    y_prob = trained_models[model_name].predict_proba(X_test)\n",
    "    # Calculate ROC curve and AUC for each class\n",
    "    for i in range(len(trained_models[model_name].classes_)):\n",
    "        fpr, tpr, _ = roc_curve(y_test == trained_models[model_name].classes_[i], y_prob[:, i])\n",
    "        roc_auc = auc(fpr, tpr)\n",
    "        \n",
    "        # Add ROC curve trace for each class\n",
    "        fig.add_trace(go.Scatter(x=fpr, y=tpr, mode='lines', \n",
    "                                 name=f'{model_name} (Class {trained_models[model_name].classes_[i]}) (AUC = {roc_auc:.2f})'))\n",
    "\n",
    "    # Add diagonal line representing random guessing\n",
    "    fig.add_trace(go.Scatter(x=[0, 1], y=[0, 1], mode='lines', line=dict(color='gray', dash='dash'), name='Random Guessing'))\n",
    "\n",
    "    fig.update_layout(title='ROC for '+model_name+' (Multi-Class)',\n",
    "                    xaxis_title='False Positive Rate',\n",
    "                    yaxis_title='True Positive Rate',\n",
    "                    xaxis_range=[0,1],yaxis_range=[0,1],\n",
    "                    legend_title='Classifiers',\n",
    "                    showlegend=True,\n",
    "                    plot_bgcolor='white', paper_bgcolor='white',\n",
    "                    xaxis=dict(linecolor='black'),\n",
    "                    yaxis=dict(linecolor='black'),\n",
    "                    legend=dict(orientation=\"h\",\n",
    "                                yanchor=\"top\",y=-0.2,\n",
    "                                xanchor=\"left\",x=-0.2),\n",
    "                    height=500, width=500)\n",
    "    fig.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}