wine - split 2 - with missing data in D1 - D2 .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "09d513f3",
   "metadata": {},
   "source": [
    "https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a90e3942",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.neighbors._base\n",
    "import sys\n",
    "sys.modules['sklearn.neighbors.base'] = sklearn.neighbors._base\n",
    "# !pip install impyute\n",
    "# !pip install fancyimpute\n",
    "from sklearn import datasets\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as skLDA\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from scipy import stats\n",
    "import numpy as np\n",
    "# import impyute as impy\n",
    "from fancyimpute import SoftImpute#, MatrixFactorization\n",
    "import pandas as pd\n",
    "import time\n",
    "import math\n",
    "# from missingpy import MissForest\n",
    "from sklearn.model_selection import train_test_split\n",
    "from numpy.linalg import norm, inv\n",
    "\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from fancyimpute import SimpleFill, SoftImpute\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "934bae58",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(178, 13)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine = datasets.load_wine()\n",
    "X,y = wine.data, wine.target.ravel() \n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa6150c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_nan(Xtrain, missing_rate):\n",
    "  Xshape = Xtrain.shape\n",
    "  na_id = np.random.randint(0,Xtrain.size,round(missing_rate*Xtrain.size))\n",
    "  Xtr_nan = Xtrain.flatten()\n",
    "  Xtr_nan[na_id] = np.nan \n",
    "  return Xtr_nan.reshape(Xshape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4929409b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_iter(clf, clf1, clf2, missing_rate):\n",
    "    X1, X2, y1, y2 = train_test_split(X,y.reshape((-1,1)),test_size=0.3)\n",
    "    Xtrain1, Xtest1, ytrain1, ytest1 = train_test_split(X1,y1.reshape((-1,1)),test_size=0.5)\n",
    "    Xtrain2, Xtest2, ytrain2, ytest2 = train_test_split(X2,y2.reshape((-1,1)),test_size=0.5)            \n",
    "\n",
    "    Xtrain1_ori = Xtrain1[:,ndel1:].copy()\n",
    "    Xtrain2_ori = Xtrain2[:,:ndel2].copy()\n",
    "    Xtest1_ori = Xtest1[:,ndel1:].copy()\n",
    "    Xtest2_ori = Xtest2[:,:ndel2].copy()    \n",
    "    \n",
    "    Xtrain1[:, :ndel1] = np.nan\n",
    "    Xtest1[:,:ndel1] = np.nan\n",
    "    Xtrain2[:, ndel2:] = np.nan\n",
    "    Xtest2[:,ndel2:] = np.nan\n",
    "    \n",
    "    Xtrain1[:,ndel1:] = generate_nan(Xtrain1[:,ndel1:] , missing_rate)\n",
    "    Xtest1[:,ndel1:] = generate_nan(Xtest1[:,ndel1:], missing_rate)\n",
    "    Xtrain2[:,:ndel2] = generate_nan(Xtrain2[:,:ndel2], missing_rate)\n",
    "    Xtest2[:,:ndel2] = generate_nan(Xtest2[:,:ndel2], missing_rate)\n",
    "    \n",
    "    Xtrain = np.vstack((Xtrain1, Xtrain2))\n",
    "    Xtest = np.vstack((Xtest1, Xtest2))\n",
    "    ytrain = np.vstack((ytrain1, ytrain2)) \n",
    "    ytest = np.vstack((ytest1, ytest2))\n",
    "    \n",
    "    Xtrain1 = Xtrain1[:,ndel1:]\n",
    "    Xtrain2 = Xtrain2[:,:ndel2]\n",
    "    Xtest1 = Xtest1[:,ndel1:]\n",
    "    Xtest2 = Xtest2[:,:ndel2]\n",
    "    \n",
    "    scaler = MinMaxScaler()\n",
    "    scaler.fit(Xtrain1)\n",
    "    Xtrain1 = scaler.transform(Xtrain1)\n",
    "    Xtrain1_ori = scaler.transform(Xtrain1_ori)\n",
    "    Xtest1 = scaler.transform(Xtest1)\n",
    "    Xtest1_ori = scaler.transform(Xtest1_ori)\n",
    "    \n",
    "    scaler.fit(Xtrain2)\n",
    "    Xtrain2 = scaler.transform(Xtrain2)\n",
    "    Xtrain2_ori = scaler.transform(Xtrain2_ori)\n",
    "    Xtest2 = scaler.transform(Xtest2)    \n",
    "    Xtest2_ori = scaler.transform(Xtest2_ori)    \n",
    "\n",
    "    scaler.fit(Xtrain)\n",
    "    Xtrain = scaler.transform(Xtrain)\n",
    "    Xtest = scaler.transform(Xtest)\n",
    "    Xtrain = SoftImpute(verbose = False).fit_transform(Xtrain)\n",
    "    Xtest = SoftImpute(verbose = False).fit_transform(Xtest)\n",
    "    Xtrain1 = SoftImpute(verbose = False).fit_transform(Xtrain1)\n",
    "    Xtrain2 = SoftImpute(verbose = False).fit_transform(Xtrain2)\n",
    "    Xtest1 = SoftImpute(verbose = False).fit_transform(Xtest1)\n",
    "    Xtest2 = SoftImpute(verbose = False).fit_transform(Xtest2)\n",
    "   \n",
    "    clf.fit(Xtrain, ytrain)\n",
    "    clf1.fit(Xtrain1, ytrain1)\n",
    "    clf2.fit(Xtrain2, ytrain2)\n",
    "        \n",
    "    acc_res = np.array([np.mean(clf.predict(Xtest[:len(ytest1),:])==ytest1.flatten()), \n",
    "                     np.mean(clf.predict(Xtest[len(ytest1):,:])==ytest2.flatten()),\n",
    "                    clf1.score(Xtest1, ytest1),  clf2.score(Xtest2, ytest2)])\n",
    "    mse_impute = np.array([np.mean((Xtrain1 - Xtrain1_ori)**2), np.mean((Xtest1 - Xtest1_ori)**2),\n",
    "                           np.mean((Xtrain[:len(Xtrain1),ndel1:] - Xtrain1_ori)**2), \n",
    "                           np.mean((Xtest[:len(Xtest1),ndel1:] - Xtest1_ori)**2),\n",
    "                           np.mean((Xtrain2 - Xtrain2_ori)**2), np.mean((Xtest2 - Xtest2_ori)**2),\n",
    "                           np.mean((Xtrain[len(Xtrain1):,:ndel2] - Xtrain2_ori)**2), \n",
    "                           np.mean((Xtest[len(Xtest1):,:ndel2] - Xtest2_ori)**2)]).reshape((-1,2))\n",
    "    return [acc_res, mse_impute]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "28a8ce9e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.90820968, 0.87474074, 0.9166129 , 0.85959259]),\n",
       " array([0.00135836, 0.00417569, 0.00127292, 0.00666848]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ndel1, ndel2 = 2, 11\n",
    "nruns = 1000\n",
    "clf = LogisticRegression(random_state=0)\n",
    "clf1 = LogisticRegression(random_state=0)\n",
    "clf2 = LogisticRegression(random_state=0)\n",
    "res = [one_iter(clf, clf1, clf2, missing_rate = .2) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f1e602b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.00793265, 0.00778459],\n",
       "        [0.01020385, 0.01021916],\n",
       "        [0.01347122, 0.01333388],\n",
       "        [0.02273571, 0.02522764]]),\n",
       " array([[2.96438059e-06, 2.98049338e-06],\n",
       "        [7.91013851e-06, 9.72243492e-06],\n",
       "        [2.08678150e-05, 2.05860949e-05],\n",
       "        [4.81097090e-05, 7.95876240e-05]]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "be9b0856",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.87737097 0.83837037 0.88356452 0.8082963 ] [0.00170067 0.00553841 0.00170631 0.00824557]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.01816288, 0.017888  ],\n",
       "        [0.02054043, 0.02052362],\n",
       "        [0.03310652, 0.03225823],\n",
       "        [0.04181108, 0.0445817 ]]),\n",
       " array([[1.05895279e-05, 1.11543723e-05],\n",
       "        [2.00408793e-05, 2.18754290e-05],\n",
       "        [8.90937313e-05, 8.27346855e-05],\n",
       "        [1.26182175e-04, 1.86147274e-04]]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .4) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b9439efd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.83112903 0.78288889 0.83464516 0.72988889] [0.00265689 0.00625097 0.00270512 0.01036098]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.03206217, 0.03213652],\n",
       "        [0.03459798, 0.03493264],\n",
       "        [0.06281139, 0.06051823],\n",
       "        [0.06876991, 0.07085559]]),\n",
       " array([[2.97356333e-05, 3.45679728e-05],\n",
       "        [4.14469527e-05, 4.80535432e-05],\n",
       "        [2.98017306e-04, 2.46721724e-04],\n",
       "        [3.23755436e-04, 3.73294097e-04]]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .6) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b4e06d5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.76154839 0.71266667 0.76072581 0.64496296] [0.00339473 0.00827357 0.00359203 0.01173197]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.0558572 , 0.05451565],\n",
       "        [0.05820464, 0.05716611],\n",
       "        [0.11443691, 0.10953645],\n",
       "        [0.11536247, 0.117818  ]]),\n",
       " array([[9.23300904e-05, 8.66914640e-05],\n",
       "        [1.08458160e-04, 9.91440606e-05],\n",
       "        [3.43350158e-03, 7.39546008e-03],\n",
       "        [3.58431275e-03, 8.37882502e-03]]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .8) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "518cc6b3",
   "metadata": {},
   "source": [
    "## SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "85738f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = make_pipeline(StandardScaler(), SVC(gamma='auto'))\n",
    "clf1 = make_pipeline(StandardScaler(), SVC(gamma='auto'))\n",
    "clf2 = make_pipeline(StandardScaler(), SVC(gamma='auto'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "510d03a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.93845161 0.89525926 0.93908065 0.88151852] [0.00084499 0.003504   0.00087806 0.00595254]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.00796429, 0.00778817],\n",
       "        [0.01035619, 0.01037752],\n",
       "        [0.01361895, 0.01302472],\n",
       "        [0.02273803, 0.02455241]]),\n",
       " array([[2.91588789e-06, 3.06120260e-06],\n",
       "        [8.61054041e-06, 1.05720650e-05],\n",
       "        [2.04653522e-05, 1.88578741e-05],\n",
       "        [4.33736773e-05, 6.81037288e-05]]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .2) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "caf8c804",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.90206452 0.84618519 0.90056452 0.81725926] [0.00138002 0.00541233 0.00140473 0.00884725]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.01832069, 0.01808816],\n",
       "        [0.02088356, 0.02090323],\n",
       "        [0.03359368, 0.03183512],\n",
       "        [0.04173581, 0.04343481]]),\n",
       " array([[1.07625205e-05, 1.20191620e-05],\n",
       "        [2.17472221e-05, 2.38887889e-05],\n",
       "        [1.00860028e-04, 8.09709682e-05],\n",
       "        [1.39867421e-04, 1.76458672e-04]]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .4) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bfd4a37b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.84554839 0.77844444 0.84187097 0.72222222] [0.00226957 0.00759626 0.00231571 0.01299588]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.03257797, 0.03232201],\n",
       "        [0.03510129, 0.0350439 ],\n",
       "        [0.06233989, 0.06050585],\n",
       "        [0.06784773, 0.07056643]]),\n",
       " array([[2.90341699e-05, 3.44646520e-05],\n",
       "        [3.94262675e-05, 4.52978710e-05],\n",
       "        [3.11100435e-04, 2.74979537e-04],\n",
       "        [3.51280210e-04, 4.30425196e-04]]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .6) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b3e21d70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.76716129 0.68566667 0.76222581 0.59448148] [0.00344482 0.00928646 0.00402483 0.01407188]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[0.05589455, 0.05504481],\n",
       "        [0.05826093, 0.05771055],\n",
       "        [0.11106524, 0.1045343 ],\n",
       "        [0.11164699, 0.11258128]]),\n",
       " array([[9.58446104e-05, 9.39613723e-05],\n",
       "        [1.14575916e-04, 1.16135100e-04],\n",
       "        [1.90441733e-03, 9.11700185e-04],\n",
       "        [1.95785018e-03, 1.32027019e-03]]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = [one_iter(clf, clf1, clf2, missing_rate = .8) for i in range(nruns)]\n",
    "acc_res = np.array([res[i][0] for i in range(nruns)])\n",
    "print(np.mean(acc_res, axis = 0), np.var(acc_res, axis = 0))\n",
    "mse_res = np.array([res[i][1] for i in range(nruns)])\n",
    "np.mean(mse_res, axis = 0), np.var(mse_res, axis = 0) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}