machine-learning/scikit-learn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate a CSV file to work with"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing test.csv\n"
     ]
    }
   ],
   "source": [
    "%%writefile test.csv\n",
    "number,group,text,positive\n",
    "110,C,people corporation problem problem think think transport science go transport place,True\n",
    "114,A,you problem government you,True\n",
    "130,B,you people place corporation problem science,True\n",
    "185,C,have think corporation fact corporation transport transport,True\n",
    "149,A,people have science science go go corporation you you fact,True\n",
    "115,C,science people go transport corporation,True\n",
    "156,A,transport corporation,True\n",
    "190,C,transport you,True\n",
    "160,A,the problem think transport place government problem go place the have place place,True\n",
    "142,B,fact science government go,True\n",
    "196,A,the fact,False\n",
    "166,A,problem the corporation you you the transport have work go work problem you,True\n",
    "168,B,have science science place the science problem,False\n",
    "167,A,government place think you corporation corporation government the go,True\n",
    "143,C,place science have fact science corporation science,True\n",
    "185,C,place the fact transport the transport work science fact people the think problem,True\n",
    "196,C,place government work science the have go work fact place the fact,True\n",
    "102,A,go government think people the have the,True\n",
    "104,C,work transport problem fact people,True\n",
    "177,C,work problem you think think you transport go think fact go go,True\n",
    "197,B,government,True\n",
    "104,A,corporation corporation fact corporation work work,True\n",
    "151,C,corporation science you people corporation government people go,True\n",
    "105,A,go fact fact,False\n",
    "165,B,think fact government the work problem work corporation transport,True\n",
    "107,B,work transport transport fact people science corporation government,True\n",
    "177,B,science people you corporation transport people corporation corporation science think have problem government,True\n",
    "186,B,you government problem you government science fact,True\n",
    "162,B,go have go go corporation fact government,True\n",
    "107,A,think have the the science problem fact think go you,False\n",
    "166,B,transport place place you go government have fact fact you the people think,True\n",
    "163,C,work problem place have the you you problem people fact science place go think,False\n",
    "121,B,place government problem,True\n",
    "118,B,work corporation people the transport corporation corporation government problem the transport have fact people,True\n",
    "106,A,the problem place work place corporation go work transport,True\n",
    "138,C,transport science,True\n",
    "111,B,fact transport corporation place corporation you place corporation you government place people transport government,True\n",
    "123,A,have work place fact science think work think go place place have problem,False\n",
    "181,C,corporation transport think,True\n",
    "156,B,have problem have transport have go the,True\n",
    "109,B,people you have have go place transport you science corporation people,True\n",
    "157,C,transport the fact have,True\n",
    "192,C,have,False\n",
    "183,C,the you problem think place think you transport people go problem fact go,True\n",
    "134,A,people place you the corporation science think government transport,True\n",
    "115,B,transport problem place problem think transport go corporation corporation you place,True\n",
    "154,A,government government think corporation have science corporation transport,True\n",
    "106,A,you go science transport problem problem the corporation problem transport problem think,True\n",
    "138,C,place transport you people work go science government transport people,True\n",
    "159,B,think have the fact think you,False\n",
    "194,B,have government the place science you,True\n",
    "127,C,work think you science science,False\n",
    "102,C,go corporation you corporation transport place place think people go fact fact,True\n",
    "103,C,have think corporation problem work go science place fact problem problem problem people,True\n",
    "127,B,corporation people think go transport fact you go transport science,True\n",
    "101,C,people science people have place corporation think,True\n",
    "149,B,the the you fact place people science problem think go think place,False\n",
    "136,A,place,False\n",
    "109,A,science problem transport,True\n",
    "156,C,work place think have work go transport you problem go,True\n",
    "168,B,fact science problem have think,False\n",
    "189,C,fact problem problem go government corporation,True\n",
    "102,B,fact the corporation have science place the have,True\n",
    "161,A,corporation you fact,True\n",
    "161,B,work corporation corporation have place work you transport science problem government,True\n",
    "177,B,fact fact,False\n",
    "123,B,government,True\n",
    "146,C,the fact you work you corporation place corporation fact,True\n",
    "136,A,work think you,False\n",
    "184,A,corporation transport go place have government go you you corporation,True\n",
    "185,A,transport go fact corporation have have government think people,True\n",
    "125,A,place place people work science think fact fact,False\n",
    "191,A,science work government fact the problem you,True\n",
    "170,B,you transport work corporation work place government people corporation you the,True\n",
    "186,A,work science think corporation think fact,True\n",
    "142,A,work people go think go fact transport go you,True\n",
    "187,A,you problem problem work,False\n",
    "183,C,corporation corporation work place fact government think work work have you have the,True\n",
    "128,A,place problem transport the,True\n",
    "118,C,transport think work you transport you work corporation have have have place,True\n",
    "151,C,work problem you go,False\n",
    "136,A,you the you think,False\n",
    "178,C,corporation you the people science people think,True\n",
    "171,C,transport you government you corporation the problem you have place place have think,True\n",
    "104,C,transport work go corporation think,True\n",
    "157,A,go place people place the,False\n",
    "114,C,science go corporation corporation government place problem the,True\n",
    "141,C,think government corporation go the government science go corporation problem place have people problem,True\n",
    "113,B,transport science fact think fact corporation corporation work,True\n",
    "170,B,transport government think you go go,True\n",
    "137,B,science people go government have,True\n",
    "170,C,science people place,False\n",
    "154,B,place go transport the government corporation fact transport fact go you corporation,True\n",
    "180,B,go transport fact government government work you science,True\n",
    "166,A,work go work people science the place people problem work go work,False\n",
    "157,C,problem fact fact the have fact go you government work people,True\n",
    "198,C,place think transport government,True\n",
    "153,A,government place problem work go the work have fact have people have work fact,True\n",
    "138,A,people,False\n",
    "111,B,transport science people think think government people fact,True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>group</th>\n",
       "      <th>text</th>\n",
       "      <th>positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110</td>\n",
       "      <td>C</td>\n",
       "      <td>people corporation problem problem think think...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114</td>\n",
       "      <td>A</td>\n",
       "      <td>you problem government you</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>130</td>\n",
       "      <td>B</td>\n",
       "      <td>you people place corporation problem science</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>185</td>\n",
       "      <td>C</td>\n",
       "      <td>have think corporation fact corporation transp...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>149</td>\n",
       "      <td>A</td>\n",
       "      <td>people have science science go go corporation ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   number group                                               text positive\n",
       "0     110     C  people corporation problem problem think think...     True\n",
       "1     114     A                         you problem government you     True\n",
       "2     130     B       you people place corporation problem science     True\n",
       "3     185     C  have think corporation fact corporation transp...     True\n",
       "4     149     A  people have science science go go corporation ...     True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True     0.78\n",
       "False    0.22\n",
       "Name: positive, dtype: float64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.positive.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the data into train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(df[df.columns.difference(['positive'])],\n",
    "                                                   df['positive'], test_size=1/3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66, 3)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66,)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34, 3)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(x_train[['number']], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76470588235294112"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Score is accuracy\n",
    "clf.score(x_test[['number']], y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True], dtype=bool)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.52644230769230771"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use `predict_proba` to get probabilities as opposed to classes\n",
    "# Get the probability for the True class\n",
    "x_test_probabilities = clf.predict_proba(x_test[['number']])[:, 1]\n",
    "roc_auc_score(y_test, x_test_probabilities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76470588235294112"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, clf.predict(x_test[['number']]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, clf.predict(x_test[['number']]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical One-Hot-Encoding (OHE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train_with_categorical = pd.get_dummies(x_train, columns=['group'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>number</th>\n",
       "      <th>text</th>\n",
       "      <th>group_A</th>\n",
       "      <th>group_B</th>\n",
       "      <th>group_C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>167</td>\n",
       "      <td>government place think you corporation corpora...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>153</td>\n",
       "      <td>government place problem work go the work have...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>141</td>\n",
       "      <td>think government corporation go the government...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110</td>\n",
       "      <td>people corporation problem problem think think...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>142</td>\n",
       "      <td>fact science government go</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    number                                               text  group_A  \\\n",
       "13     167  government place think you corporation corpora...        1   \n",
       "97     153  government place problem work go the work have...        1   \n",
       "87     141  think government corporation go the government...        0   \n",
       "0      110  people corporation problem problem think think...        0   \n",
       "9      142                         fact science government go        0   \n",
       "\n",
       "    group_B  group_C  \n",
       "13        0        0  \n",
       "97        0        0  \n",
       "87        0        1  \n",
       "0         0        1  \n",
       "9         1        0  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_with_categorical.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test_with_categorical = pd.get_dummies(x_test, columns=['group'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(x_train_with_categorical[x_train_with_categorical.columns.difference(['text'])], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76470588235294112"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(x_test_with_categorical[x_test_with_categorical.columns.difference(['text'])], y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_vectorizer = CountVectorizer()\n",
    "x_train_vectorizer.fit(x_train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_vectorized = x_train_vectorizer.transform(x_train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<66x14 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 398 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(x_train_vectorized, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82352941176470584"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(x_train_vectorizer.transform(x_test['text']), y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Same process can be repeated with TF-IDF and Hashing vectorizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combining features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<66x14 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 398 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features = np.concatenate([x_train_vectorized.todense(),\n",
    "                           x_train_with_categorical[x_test_with_categorical.columns.difference(['text'])].values\n",
    "                          ],\n",
    "                          axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[  2,   0,   1, ...,   0,   0, 167],\n",
       "        [  0,   2,   1, ...,   0,   0, 153],\n",
       "        [  2,   0,   2, ...,   0,   1, 141],\n",
       "        ..., \n",
       "        [  0,   0,   0, ...,   1,   0, 197],\n",
       "        [  0,   0,   2, ...,   1,   0, 170],\n",
       "        [  1,   0,   0, ...,   0,   1, 181]])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_test = np.concatenate([x_train_vectorizer.transform(x_test['text']).todense(),\n",
    "                                x_test_with_categorical[x_test_with_categorical.columns.difference(['text'])].values\n",
    "                               ], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.82352941176470584"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(features_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Other learners"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just swap the classifier, interface is the same\n",
    "\n",
    "List of scikit-learn classifiers: http://scikit-learn.org/stable/supervised_learning.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = RandomForestClassifier()\n",
    "clf.fit(features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76470588235294112"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(features_test, y_test)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}