Table 5 - wine - split 2 - with missing data in D1 - D2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "09d513f3",
   "metadata": {},
   "source": [
    "https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a90e3942",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.neighbors._base\n",
    "import sys\n",
    "sys.modules['sklearn.neighbors.base'] = sklearn.neighbors._base\n",
    "# !pip install impyute\n",
    "# !pip install fancyimpute\n",
    "from sklearn import datasets\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as skLDA\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from scipy import stats\n",
    "import numpy as np\n",
    "# import impyute as impy\n",
    "from fancyimpute import SoftImpute#, MatrixFactorization\n",
    "import pandas as pd\n",
    "import time\n",
    "import math\n",
    "# from missingpy import MissForest\n",
    "from sklearn.model_selection import train_test_split\n",
    "from numpy.linalg import norm, inv\n",
    "\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from fancyimpute import SimpleFill, SoftImpute\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "934bae58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(178, 13)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine = datasets.load_wine()\n",
    "X,y = wine.data, wine.target.ravel() \n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa6150c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_nan(Xtrain, missing_rate):\n",
    "  Xshape = Xtrain.shape\n",
    "  na_id = np.random.randint(0,Xtrain.size,round(missing_rate*Xtrain.size))\n",
    "  Xtr_nan = Xtrain.flatten()\n",
    "  Xtr_nan[na_id] = np.nan \n",
    "  return Xtr_nan.reshape(Xshape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4929409b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_iter(clf, clf1, clf2, missing_rate):\n",
    "    X1, X2, y1, y2 = train_test_split(X,y.reshape((-1,1)),test_size=0.3)\n",
    "    Xtrain1, Xtest1, ytrain1, ytest1 = train_test_split(X1,y1.reshape((-1,1)),test_size=0.5)\n",
    "    Xtrain2, Xtest2, ytrain2, ytest2 = train_test_split(X2,y2.reshape((-1,1)),test_size=0.5)            \n",
    "\n",
    "    Xtrain1_ori = Xtrain1[:,ndel1:].copy()\n",
    "    Xtrain2_ori = Xtrain2[:,:ndel2].copy()\n",
    "    Xtest1_ori = Xtest1[:,ndel1:].copy()\n",
    "    Xtest2_ori = Xtest2[:,:ndel2].copy()    \n",
    "    \n",
    "    Xtrain1[:, :ndel1] = np.nan\n",
    "    Xtest1[:,:ndel1] = np.nan\n",
    "    Xtrain2[:, ndel2:] = np.nan\n",
    "    Xtest2[:,ndel2:] = np.nan\n",
    "    \n",
    "    Xtrain1[:,ndel1:] = generate_nan(Xtrain1[:,ndel1:] , missing_rate)\n",
    "    Xtest1[:,ndel1:] = generate_nan(Xtest1[:,ndel1:], missing_rate)\n",
    "    Xtrain2[:,:ndel2] = generate_nan(Xtrain2[:,:ndel2], missing_rate)\n",
    "    Xtest2[:,:ndel2] = generate_nan(Xtest2[:,:ndel2], missing_rate)\n",
    "    \n",
    "    Xtrain = np.vstack((Xtrain1, Xtrain2))\n",
    "    Xtest = np.vstack((Xtest1, Xtest2))\n",
    "    ytrain = np.vstack((ytrain1, ytrain2)) \n",
    "    ytest = np.vstack((ytest1, ytest2))\n",
    "    \n",
    "    Xtrain1 = Xtrain1[:,ndel1:]\n",
    "    Xtrain2 = Xtrain2[:,:ndel2]\n",
    "    Xtest1 = Xtest1[:,ndel1:]\n",
    "    Xtest2 = Xtest2[:,:ndel2]\n",
    "    \n",
    "    scaler = MinMaxScaler()\n",
    "    scaler.fit(Xtrain1)\n",
    "    Xtrain1 = scaler.transform(Xtrain1)\n",
    "    Xtrain1_ori = scaler.transform(Xtrain1_ori)\n",
    "    Xtest1 = scaler.transform(Xtest1)\n",
    "    Xtest1_ori = scaler.transform(Xtest1_ori)\n",
    "    \n",
    "    scaler.fit(Xtrain2)\n",
    "    Xtrain2 = scaler.transform(Xtrain2)\n",
    "    Xtrain2_ori = scaler.transform(Xtrain2_ori)\n",
    "    Xtest2 = scaler.transform(Xtest2)    \n",
    "    Xtest2_ori = scaler.transform(Xtest2_ori)    \n",
    "\n",
    "    scaler.fit(Xtrain)\n",
    "    Xtrain = scaler.transform(Xtrain)\n",
    "    Xtest = scaler.transform(Xtest)\n",
    "    Xtrain = SoftImpute(verbose = False).fit_transform(Xtrain)\n",
    "    Xtest = SoftImpute(verbose = False).fit_transform(Xtest)\n",
    "    Xtrain1 = SoftImpute(verbose = False).fit_transform(Xtrain1)\n",
    "    Xtrain2 = SoftImpute(verbose = False).fit_transform(Xtrain2)\n",
    "    Xtest1 = SoftImpute(verbose = False).fit_transform(Xtest1)\n",
    "    Xtest2 = SoftImpute(verbose = False).fit_transform(Xtest2)\n",
    "   \n",
    "    clf.fit(Xtrain, ytrain)\n",
    "    clf1.fit(Xtrain1, ytrain1)\n",
    "    clf2.fit(Xtrain2, ytrain2)\n",
    "        \n",
    "    acc_res = np.array([np.mean(clf.predict(Xtest[:len(ytest1),:])==ytest1.flatten()), \n",
    "                     np.mean(clf.predict(Xtest[len(ytest1):,:])==ytest2.flatten()),\n",
    "                    clf1.score(Xtest1, ytest1),  clf2.score(Xtest2, ytest2)])\n",
    "    mse_impute = np.array([np.mean((Xtrain1 - Xtrain1_ori)**2), np.mean((Xtest1 - Xtest1_ori)**2),\n",
    "                           np.mean((Xtrain[:len(Xtrain1),ndel1:] - Xtrain1_ori)**2), \n",
    "                           np.mean((Xtest[:len(Xtest1),ndel1:] - Xtest1_ori)**2),\n",
    "                           np.mean((Xtrain2 - Xtrain2_ori)**2), np.mean((Xtest2 - Xtest2_ori)**2),\n",
    "                           np.mean((Xtrain[len(Xtrain1):,:ndel2] - Xtrain2_ori)**2), \n",
    "                           np.mean((Xtest[len(Xtest1):,:ndel2] - Xtest2_ori)**2)]).reshape((-1,2))\n",
    "    return [acc_res, mse_impute]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "28a8ce9e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.90795806, 0.87422222, 0.91710806, 0.8591    ]),\n",
       " array([0.00131855, 0.00415169, 0.00124456, 0.00669575]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ndel1, ndel2 = 2, 11\n",
    "nruns = 10000\n",
    "clf = LogisticRegression(random_state=0)\n",
    "clf1 = LogisticRegression(random_state=0)\n",
    "clf2 = LogisticRegression(random_state=0)\n",
    "res = [one_iter(clf, clf1, clf2, missing_rate = .2) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f1e602b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.00794043, 0.00784552],\n",
       "        [0.01027027, 0.01037364],\n",
       "        [0.01358377, 0.01318194],\n",
       "        [0.02290592, 0.02496911]]),\n",
       " array([[3.03746636e-06, 3.11351824e-06],\n",
       "        [8.48400229e-06, 9.83007042e-06],\n",
       "        [2.13393999e-05, 2.03046385e-05],\n",
       "        [4.86302760e-05, 7.49048651e-05]]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "be9b0856",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.87748548 0.8370963  0.88345806 0.8041037 ] [0.00179443 0.00510655 0.00174425 0.00863671]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.01817565, 0.01801088],\n",
       "        [0.0205268 , 0.02059626],\n",
       "        [0.03325116, 0.03208893],\n",
       "        [0.04174655, 0.0438298 ]]),\n",
       " array([[1.10009621e-05, 1.15289122e-05],\n",
       "        [1.91351775e-05, 2.10345009e-05],\n",
       "        [9.51391391e-05, 8.98332811e-05],\n",
       "        [1.40145229e-04, 1.83197086e-04]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .4) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b9439efd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.83094516 0.78253333 0.8338871  0.73263704] [0.002483   0.00670935 0.00248756 0.01041773]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.03240147, 0.03215747],\n",
       "        [0.03503256, 0.03499877],\n",
       "        [0.06313572, 0.06047865],\n",
       "        [0.0688551 , 0.07090001]]),\n",
       " array([[3.08087593e-05, 3.32632230e-05],\n",
       "        [4.36390379e-05, 4.75050899e-05],\n",
       "        [3.83768833e-04, 3.03621943e-04],\n",
       "        [4.43396147e-04, 4.88410000e-04]]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .6) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b4e06d5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.76249516 0.71198889 0.76229194 0.64508519] [0.00344678 0.00845934 0.00367237 0.01174041]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.05569442, 0.05493329],\n",
       "        [0.05807017, 0.05761648],\n",
       "        [0.11046284, 0.10555399],\n",
       "        [0.11138044, 0.1133759 ]]),\n",
       " array([[9.22047271e-05, 8.97498311e-05],\n",
       "        [1.11038832e-04, 1.08137669e-04],\n",
       "        [1.77381994e-03, 1.49452411e-03],\n",
       "        [1.80142293e-03, 1.95252880e-03]]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .8) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "518cc6b3",
   "metadata": {},
   "source": [
    "## SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "85738f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))\n",
    "clf1 = make_pipeline(StandardScaler(), SVC(gamma='auto'))\n",
    "clf2 = make_pipeline(StandardScaler(), SVC(gamma='auto'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "510d03a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.93919839 0.89321481 0.93916129 0.88215556] [0.00088201 0.00404905 0.00091509 0.00608361]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.00793888, 0.00784598],\n",
       "        [0.01027703, 0.01038809],\n",
       "        [0.01365136, 0.01326882],\n",
       "        [0.02299103, 0.02506474]]),\n",
       " array([[3.03448411e-06, 3.05705377e-06],\n",
       "        [8.71261816e-06, 1.00641694e-05],\n",
       "        [2.24823428e-05, 2.09624282e-05],\n",
       "        [5.05144950e-05, 7.70761364e-05]]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .2) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "caf8c804",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.90097903 0.84629259 0.90050484 0.814     ] [0.00146683 0.00549653 0.00147889 0.00908905]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.01823769, 0.01800547],\n",
       "        [0.02060988, 0.0206098 ],\n",
       "        [0.03322545, 0.03194564],\n",
       "        [0.041758  , 0.04375048]]),\n",
       " array([[1.08378373e-05, 1.16481573e-05],\n",
       "        [1.92904187e-05, 2.18225549e-05],\n",
       "        [9.22043484e-05, 8.60600817e-05],\n",
       "        [1.39381090e-04, 1.82370480e-04]]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .4) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bfd4a37b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8457     0.7788037  0.84301613 0.71567778] [0.00222299 0.0078257  0.0022851  0.01238488]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.03247349, 0.03210667],\n",
       "        [0.03510671, 0.03494635],\n",
       "        [0.06310183, 0.06067863],\n",
       "        [0.06879191, 0.07088789]]),\n",
       " array([[3.10379988e-05, 3.28553116e-05],\n",
       "        [4.40765273e-05, 4.71320477e-05],\n",
       "        [3.76711804e-04, 3.03344592e-04],\n",
       "        [4.42933810e-04, 4.72296372e-04]]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .6) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b3e21d70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.7664371  0.69224074 0.76160806 0.60308889] [0.00336281 0.00961359 0.00364741 0.01399353]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.055751  , 0.05502999],\n",
       "        [0.05811748, 0.05766407],\n",
       "        [0.11118633, 0.10573754],\n",
       "        [0.1120555 , 0.11378811]]),\n",
       " array([[9.02459867e-05, 8.99581923e-05],\n",
       "        [1.08891595e-04, 1.08066614e-04],\n",
       "        [2.19766511e-03, 1.22274731e-03],\n",
       "        [2.27248160e-03, 1.81694813e-03]]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .8) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}