xgb_fourier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = \"3\"\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Dense, Dropout, TimeDistributed\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from tqdm import tqdm_notebook, tqdm\n",
    "from IPython.display import clear_output, display\n",
    "from keras import backend as K\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.cross_validation import cross_val_score, train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler, RobustScaler, Normalizer\n",
    "from sklearn import manifold, decomposition, linear_model, ensemble, neighbors, cross_validation\n",
    "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
    "\n",
    "from scipy.fftpack import ifftn, fftn\n",
    "\n",
    "from scipy import stats\n",
    "from glob import glob\n",
    "from multiprocessing import Pool\n",
    "from sklearn.metrics import roc_auc_score as auc\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('train_labels.csv')\n",
    "data['train'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500, 560)\n"
     ]
    }
   ],
   "source": [
    "functions = [np.max, np.min, np.std, np.mean, np.median, fftn]\n",
    "n_size = 5\n",
    "\n",
    "\n",
    "def calc_feats(filename):\n",
    "    #data = np.load(filename)\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    v = []\n",
    "    for func in functions:\n",
    "        if func == fftn:\n",
    "            d = fftn(data, shape=(n_size,), axes=(0,))\n",
    "            d = np.real(d)\n",
    "            d = d.ravel()\n",
    "            v.extend(list(d))\n",
    "        else:\n",
    "            v.extend(list(func(data, axis=0)))\n",
    "    return v\n",
    "\n",
    "files = ['/mnt/kaspersky/data_kasp/train/'+f for f in data.SeriesId][:]\n",
    "\n",
    "pool = Pool(16)\n",
    "X = None\n",
    "try: X = pd.DataFrame(pool.map(calc_feats, files))\n",
    "except ValueError as error: print(error)\n",
    "pool.terminate()\n",
    "X.fillna(-999,inplace=True)\n",
    "y = data.copy()[:len(X)]\n",
    "print X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.993136\teval-auc:0.798584\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 70 rounds.\n",
      "Stopping. Best iteration:\n",
      "[119]\ttrain-auc:1\teval-auc:0.853282\n",
      "\n",
      "[0]\ttrain-auc:0.960586\teval-auc:0.834191\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 70 rounds.\n",
      "Stopping. Best iteration:\n",
      "[4]\ttrain-auc:0.998043\teval-auc:0.908408\n",
      "\n",
      "[0]\ttrain-auc:0.982572\teval-auc:0.779923\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 70 rounds.\n",
      "Stopping. Best iteration:\n",
      "[195]\ttrain-auc:1\teval-auc:0.871729\n",
      "\n",
      "[0]\ttrain-auc:0.982384\teval-auc:0.826684\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 70 rounds.\n",
      "Stopping. Best iteration:\n",
      "[123]\ttrain-auc:1\teval-auc:0.9142\n",
      "\n",
      "[0]\ttrain-auc:0.964098\teval-auc:0.834406\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 70 rounds.\n",
      "Stopping. Best iteration:\n",
      "[8]\ttrain-auc:0.999035\teval-auc:0.909052\n",
      "\n",
      "XGB: 0.884 +- 0.020\n",
      "ET: 0.903 +- 0.023\n"
     ]
    }
   ],
   "source": [
    "param = {}\n",
    "param['objective'] = 'binary:logistic'\n",
    "param['eval_metric'] = 'auc'\n",
    "\n",
    "param['max_depth'] = 7\n",
    "\n",
    "param['booster'] = 'gbtree'\n",
    "# param['booster'] = 'gblinear'\n",
    "param['eta'] = 0.005\n",
    "# param['subsample'] = 0.6\n",
    "param['colsample_bytree'] = 0.3\n",
    "param['nthread'] = 8\n",
    "#param['alpha'] = 0\n",
    "param['min_child_weight'] = 1\n",
    "\n",
    "# if param['booster'] == 'gblinear': param['eta'] *= 1e-4\n",
    "numround = 15001\n",
    "# scores, , []\n",
    "\n",
    "skf = StratifiedKFold(5, shuffle=True, random_state=0)\n",
    "sc,sc2 = [],[]\n",
    "pred_train1 = np.zeros(len(y))\n",
    "pred_train2 = np.zeros(len(y))\n",
    "for itr, ite in skf.split(y[y['train'] == 1]['SeriesId'].values, y[y['train'] == 1]['Attack'].values):\n",
    "\n",
    "    ypred,ypred2 = [],[]\n",
    "    for i in range(1):\n",
    "        param['seed'] = i+1\n",
    "\n",
    "        Xdatatrain = xgboost.DMatrix(data = X.iloc[itr].values,\n",
    "                                     label = y.iloc[itr]['Attack'].values)\n",
    "        Xdataval = xgboost.DMatrix(data = X.iloc[ite].values,\n",
    "                                    label = y.iloc[ite]['Attack'].values)\n",
    "\n",
    "        plst = list(param.items())\n",
    "        watchlist = [(Xdatatrain, 'train'), (Xdataval, 'eval')]\n",
    "        bst = xgboost.train(plst, Xdatatrain, 100000, evals=watchlist, verbose_eval=1000, \n",
    "                            early_stopping_rounds=int(5 / param['eta'] ** 0.5))\n",
    "\n",
    "        ypred.append(bst.predict(Xdataval))\n",
    "        \n",
    "        clf = ExtraTreesClassifier(n_estimators=1000, n_jobs=16, criterion='entropy',\n",
    "                                  max_features=0.1, min_samples_split=10)\n",
    "        #clf = LogisticRegression(C=1)\n",
    "        clf.fit(X.iloc[itr].values,y.iloc[itr]['Attack'].values)\n",
    "        ypred2.append(clf.predict_proba(X.iloc[ite].values)[:,1])\n",
    "\n",
    "    ypred = sum(ypred)/len(ypred)\n",
    "    ypred2 = sum(ypred2)/len(ypred2)\n",
    "    pred_train1[ite] = ypred\n",
    "    pred_train2[ite] = ypred2\n",
    "    \n",
    "    sc.append(auc(y.iloc[ite]['Attack'].values, ypred))\n",
    "    sc2.append(auc(y.iloc[ite]['Attack'].values, ypred2))\n",
    "#     break\n",
    "\n",
    "print( 'XGB: {:.3f} +- {:.3f}'.format(np.mean(sc), np.std(sc)))\n",
    "print('ET: {:.3f} +- {:.3f}'.format(np.mean(sc2), np.std(sc2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}