name_matching_tfidf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Company Name Matching\n",
    "### Approach\n",
    "1. Preprocess to noramlize names: remove spaces, characters, lowercase, \n",
    "2. Build TfIdf name vectors\n",
    "3. Index Using [Annoy](https://github.com/spotify/annoy)\n",
    "4. Evaluate Using accuracy on non -1 companies.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn import svm, linear_model\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import roc_curve, auc,f1_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from collections import defaultdict\n",
    "import unicodedata\n",
    "import random\n",
    "random.seed(37)\n",
    "from annoy import AnnoyIndex\n",
    "import nltk\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"data/\"\n",
    "file_path = \"files/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = pd.read_csv(data_path+\"G.csv\", delimiter=\"|\")\n",
    "train = pd.read_csv(data_path+\"STrain.csv\", delimiter=\"|\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>634022</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>324497</td>\n",
       "      <td>The David Isaacs Fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>280848</td>\n",
       "      <td>Bramor Enterprises Limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>432662</td>\n",
       "      <td>NAVEXIM S.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>524224</td>\n",
       "      <td>Magal Group SA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company_id                        name\n",
       "0  634022      PRIMCOM SA                \n",
       "1  324497      The David Isaacs Fund     \n",
       "2  280848      Bramor Enterprises Limited\n",
       "3  432662      NAVEXIM S.A.              \n",
       "4  524224      Magal Group SA            "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 450256 entries, 0 to 450255\n",
      "Data columns (total 2 columns):\n",
      "company_id    450256 non-null int64\n",
      "name          450256 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 6.9+ MB\n"
     ]
    }
   ],
   "source": [
    "companies.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA</td>\n",
       "      <td>177358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A IRL Fuund</td>\n",
       "      <td>568472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMR-500 Kendall LLC 1 Mezz GmbH</td>\n",
       "      <td>195692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Solich GmbH KG</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Drzyzzga Funds Logi. sp. z oo</td>\n",
       "      <td>404178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index                                                          name  \\\n",
       "0  0            TRATTAMENTO Ltd RIFIUTI METROPOLITANI SPA  SIGLABILE TRM SPA   \n",
       "1  1            A IRL Fuund                                                    \n",
       "2  2            BMR-500 Kendall LLC 1 Mezz GmbH                                \n",
       "3  3            Solich GmbH KG                                                 \n",
       "4  4            Drzyzzga Funds Logi. sp. z oo                                  \n",
       "\n",
       "   company_id  \n",
       "0  177358      \n",
       "1  568472      \n",
       "2  195692      \n",
       "3 -1           \n",
       "4  404178      "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100000 entries, 0 to 99999\n",
      "Data columns (total 3 columns):\n",
      "train_index    100000 non-null int64\n",
      "name           100000 non-null object\n",
      "company_id     100000 non-null int64\n",
      "dtypes: int64(2), object(1)\n",
      "memory usage: 2.3+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_index</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>THEking'S ROYAL HUSSARS OFFI. TRUST' TRUST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Southern Powe rcompany SICAV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BMO S&amp;P/TSX Ladde. Share ETF Index</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>PaI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Clearview Two</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   test_index                                        name\n",
       "0  0           THEking'S ROYAL HUSSARS OFFI. TRUST' TRUST\n",
       "1  1           Southern Powe rcompany SICAV              \n",
       "2  2           BMO S&P/TSX Ladde. Share ETF Index        \n",
       "3  3           PaI                                       \n",
       "4  4           Clearview Two                             "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(data_path+\"STest.csv\", delimiter=\",\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100000 entries, 0 to 99999\n",
      "Data columns (total 2 columns):\n",
      "test_index    100000 non-null int64\n",
      "name          99999 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 1.5+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'train_index', u'name', u'company_id'], dtype='object')\n",
      "Index([u'test_index', u'name'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print train.columns\n",
    "print test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 3)\n",
      "99365\n"
     ]
    }
   ],
   "source": [
    "print train.shape\n",
    "print train[\"name\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 2)\n",
      "99323\n"
     ]
    }
   ],
   "source": [
    "print test.shape\n",
    "print test[\"name\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1075\n"
     ]
    }
   ],
   "source": [
    "print len(set(test[\"name\"]) & set(train[\"name\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample Data\n",
    "Here I use tf-idf vectors. Tfidf vectors are sparse and have a very high dimesnion ~15,000. Though I have used scipy sparse matrices, I cannot get sparse format to work with Annoy. Consequently I used the dense format to create indexex. This increased the index size and it became unmanagable. As a result I decided to work with only a sample of data.\n",
    "\n",
    "1. Sample 50,000 (fraction =1/9) companies from the original company set.\n",
    "2. To build train sample:\n",
    "    * select all records with company id prsent in companies sample\n",
    "    * Take 1/9 fraction of -1 companies "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = companies.sample(50000, random_state=37)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>619393</td>\n",
       "      <td>Metzler Vermögensverwaltungsfonds 70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>587104</td>\n",
       "      <td>Hakvoort Professional B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>482637</td>\n",
       "      <td>RUDOLF WOLFF SYSTEMATIC FUND LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>301258</td>\n",
       "      <td>Western Alliance Bancorporation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>499801</td>\n",
       "      <td>Ivar Fors &amp; Co AB</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company_id                                  name\n",
       "0  619393      Metzler Vermögensverwaltungsfonds 70\n",
       "1  587104      Hakvoort Professional B.V.          \n",
       "2  482637      RUDOLF WOLFF SYSTEMATIC FUND LIMITED\n",
       "3  301258      Western Alliance Bancorporation     \n",
       "4  499801      Ivar Fors & Co AB                   "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies = companies.reset_index(drop=True)\n",
    "companies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30256, 3)\n",
      "(69744, 3)\n"
     ]
    }
   ],
   "source": [
    "train_0 = train.loc[train[\"company_id\"]==-1]\n",
    "print train_0.shape\n",
    "train_1 = train.loc[train[\"company_id\"]>=0]\n",
    "\n",
    "print train_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7708, 3)\n"
     ]
    }
   ],
   "source": [
    "train = train_1.loc[train_1[\"company_id\"].isin(companies[\"company_id\"])]\n",
    "print train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3361\n"
     ]
    }
   ],
   "source": [
    "train0_sample = train_0.sample(int((1/9.0)*len(train_0)), random_state=37)\n",
    "print len(train0_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train,train0_sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11069, 3)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>BERK ELEY LLP</td>\n",
       "      <td>438615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>RBPA Leeuw</td>\n",
       "      <td>526042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19</td>\n",
       "      <td>Waddell &amp; Reed Advisors Funds - Waddell &amp; Reed Advisors Small Cap Fund</td>\n",
       "      <td>281405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>ESCO TORE S.R.L.</td>\n",
       "      <td>79550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>Russell Total Return Fund.) Ltd.</td>\n",
       "      <td>351412</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index  \\\n",
       "0  5             \n",
       "1  9             \n",
       "2  19            \n",
       "3  22            \n",
       "4  72            \n",
       "\n",
       "                                                                     name  \\\n",
       "0  BERK ELEY LLP                                                            \n",
       "1  RBPA Leeuw                                                               \n",
       "2  Waddell & Reed Advisors Funds - Waddell & Reed Advisors Small Cap Fund   \n",
       "3  ESCO TORE S.R.L.                                                         \n",
       "4  Russell Total Return Fund.) Ltd.                                         \n",
       "\n",
       "   company_id  \n",
       "0  438615      \n",
       "1  526042      \n",
       "2  281405      \n",
       "3  79550       \n",
       "4  351412      "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.reset_index(drop=True)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_accents(df,**kw_args):\n",
    "    old_col =  kw_args[\"old_col\"]\n",
    "    new_col =  kw_args[\"new_col\"]\n",
    "    def remove_accents_inner(input_str):\n",
    "        nfkd_form = unicodedata.normalize('NFKD', unicode(input_str, 'utf8'))\n",
    "        return u\"\".join([c for c in nfkd_form if not unicodedata.combining(c)])\n",
    "    \n",
    "    df[new_col] = df[old_col].apply(remove_accents_inner)\n",
    "    return df\n",
    "\n",
    "def make_accent_transformer(old_col, new_col):\n",
    "    return FunctionTransformer(remove_accents, validate=False,\n",
    "                                         kw_args={\"old_col\":old_col,\"new_col\":new_col})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/akash/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]|@,;\\-\\\\\\+#~!$%^]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z ]')\n",
    "STOPWORDS = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "india add g ddcbg c g hh ak ash sin gh dee pt su\n"
     ]
    }
   ],
   "source": [
    "s = \"for from india and add+g-ddc.b.g,c\\g-hh ak|ash#sin/gh dee$pt^a su%d\"\n",
    "s=re.sub(REPLACE_BY_SPACE_RE,\" \",s)\n",
    "s= re.sub(BAD_SYMBOLS_RE,\"\",s)\n",
    "s= ' '.join(word for word in s.split() if word not in STOPWORDS)\n",
    "print s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_name(df,**kw_args):\n",
    "    old_col =  kw_args[\"old_col\"]\n",
    "    new_col =  kw_args[\"new_col\"]\n",
    "    \n",
    "    def regex_clean(text):\n",
    "        text = re.sub(REPLACE_BY_SPACE_RE, \" \", text)\n",
    "        text = re.sub(BAD_SYMBOLS_RE, \"\", text)\n",
    "        #text= ' '.join(word for word in text.split() if word not in STOPWORDS)\n",
    "        return text\n",
    "    \n",
    "    df[new_col] = df[old_col].str.lower().str.strip()\n",
    "    df[new_col] = df[old_col].apply(regex_clean)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def make_clean_name_transformer(old_col, new_col):\n",
    "    return FunctionTransformer(clean_name, validate=False, kw_args={\"old_col\":old_col,\"new_col\":new_col})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make name processing pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_pipeline = make_pipeline(make_accent_transformer(\"name\", \"clean_name\"),\n",
    "                                    make_clean_name_transformer(\"clean_name\", \"clean_name\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37157</th>\n",
       "      <td>78742</td>\n",
       "      <td>PowerShares S&amp;P 500 High Dividend Low Volatility Index ETF</td>\n",
       "      <td>powershares sp 500 high dividend low volatility index etf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15724</th>\n",
       "      <td>571443</td>\n",
       "      <td>Bauschutz GmbH &amp; Co. KG</td>\n",
       "      <td>bauschutz gmbh  co kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32459</th>\n",
       "      <td>50788</td>\n",
       "      <td>DAJIMA Holding B.V.</td>\n",
       "      <td>dajima holding bv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23109</th>\n",
       "      <td>435011</td>\n",
       "      <td>A.J. BRUNT LIMITED</td>\n",
       "      <td>aj brunt limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37703</th>\n",
       "      <td>244562</td>\n",
       "      <td>FLEX.DOCCIA S.R.L.</td>\n",
       "      <td>flexdoccia srl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33840</th>\n",
       "      <td>254844</td>\n",
       "      <td>Pacific Funds Series Trust - PF Mid-Cap Value Fund</td>\n",
       "      <td>pacific funds series trust   pf mid cap value fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4904</th>\n",
       "      <td>400061</td>\n",
       "      <td>Apotheke zur Post Inh. Anette Penz e.K</td>\n",
       "      <td>apotheke zur post inh anette penz ek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26736</th>\n",
       "      <td>59118</td>\n",
       "      <td>Clipper Oil</td>\n",
       "      <td>clipper oil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31961</th>\n",
       "      <td>156980</td>\n",
       "      <td>T. Rowe Price Funds SICAV - Global Focused Growth Equity Fund</td>\n",
       "      <td>t rowe price funds sicav   global focused growth equity fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24117</th>\n",
       "      <td>472930</td>\n",
       "      <td>Metropolitan Tower Life Insurance Company</td>\n",
       "      <td>metropolitan tower life insurance company</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       company_id  \\\n",
       "37157  78742        \n",
       "15724  571443       \n",
       "32459  50788        \n",
       "23109  435011       \n",
       "37703  244562       \n",
       "33840  254844       \n",
       "4904   400061       \n",
       "26736  59118        \n",
       "31961  156980       \n",
       "24117  472930       \n",
       "\n",
       "                                                                name  \\\n",
       "37157  PowerShares S&P 500 High Dividend Low Volatility Index ETF      \n",
       "15724  Bauschutz GmbH & Co. KG                                         \n",
       "32459  DAJIMA Holding B.V.                                             \n",
       "23109  A.J. BRUNT LIMITED                                              \n",
       "37703  FLEX.DOCCIA S.R.L.                                              \n",
       "33840  Pacific Funds Series Trust - PF Mid-Cap Value Fund              \n",
       "4904   Apotheke zur Post Inh. Anette Penz e.K                          \n",
       "26736  Clipper Oil                                                     \n",
       "31961  T. Rowe Price Funds SICAV - Global Focused Growth Equity Fund   \n",
       "24117  Metropolitan Tower Life Insurance Company                       \n",
       "\n",
       "                                                         clean_name  \n",
       "37157  powershares sp 500 high dividend low volatility index etf     \n",
       "15724  bauschutz gmbh  co kg                                         \n",
       "32459  dajima holding bv                                             \n",
       "23109  aj brunt limited                                              \n",
       "37703  flexdoccia srl                                                \n",
       "33840  pacific funds series trust   pf mid cap value fund            \n",
       "4904   apotheke zur post inh anette penz ek                          \n",
       "26736  clipper oil                                                   \n",
       "31961  t rowe price funds sicav   global focused growth equity fund  \n",
       "24117  metropolitan tower life insurance company                     "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies = name_pipeline.transform(companies)\n",
    "companies.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 3)\n",
      "49963\n"
     ]
    }
   ],
   "source": [
    "print companies.shape\n",
    "print companies[\"clean_name\"].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pre-process train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>BERK ELEY LLP</td>\n",
       "      <td>438615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>RBPA Leeuw</td>\n",
       "      <td>526042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19</td>\n",
       "      <td>Waddell &amp; Reed Advisors Funds - Waddell &amp; Reed Advisors Small Cap Fund</td>\n",
       "      <td>281405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>ESCO TORE S.R.L.</td>\n",
       "      <td>79550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>Russell Total Return Fund.) Ltd.</td>\n",
       "      <td>351412</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index  \\\n",
       "0  5             \n",
       "1  9             \n",
       "2  19            \n",
       "3  22            \n",
       "4  72            \n",
       "\n",
       "                                                                     name  \\\n",
       "0  BERK ELEY LLP                                                            \n",
       "1  RBPA Leeuw                                                               \n",
       "2  Waddell & Reed Advisors Funds - Waddell & Reed Advisors Small Cap Fund   \n",
       "3  ESCO TORE S.R.L.                                                         \n",
       "4  Russell Total Return Fund.) Ltd.                                         \n",
       "\n",
       "   company_id  \n",
       "0  438615      \n",
       "1  526042      \n",
       "2  281405      \n",
       "3  79550       \n",
       "4  351412      "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11069, 3)\n"
     ]
    }
   ],
   "source": [
    "print train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = name_pipeline.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>BERK ELEY LLP</td>\n",
       "      <td>438615</td>\n",
       "      <td>berk eley llp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>RBPA Leeuw</td>\n",
       "      <td>526042</td>\n",
       "      <td>rbpa leeuw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19</td>\n",
       "      <td>Waddell &amp; Reed Advisors Funds - Waddell &amp; Reed Advisors Small Cap Fund</td>\n",
       "      <td>281405</td>\n",
       "      <td>waddell  reed advisors funds   waddell  reed advisors small cap fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>ESCO TORE S.R.L.</td>\n",
       "      <td>79550</td>\n",
       "      <td>esco tore srl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>Russell Total Return Fund.) Ltd.</td>\n",
       "      <td>351412</td>\n",
       "      <td>russell total return fund  ltd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_index  \\\n",
       "0  5             \n",
       "1  9             \n",
       "2  19            \n",
       "3  22            \n",
       "4  72            \n",
       "\n",
       "                                                                     name  \\\n",
       "0  BERK ELEY LLP                                                            \n",
       "1  RBPA Leeuw                                                               \n",
       "2  Waddell & Reed Advisors Funds - Waddell & Reed Advisors Small Cap Fund   \n",
       "3  ESCO TORE S.R.L.                                                         \n",
       "4  Russell Total Return Fund.) Ltd.                                         \n",
       "\n",
       "   company_id  \\\n",
       "0  438615       \n",
       "1  526042       \n",
       "2  281405       \n",
       "3  79550        \n",
       "4  351412       \n",
       "\n",
       "                                                             clean_name  \n",
       "0  berk eley llp                                                         \n",
       "1  rbpa leeuw                                                            \n",
       "2  waddell  reed advisors funds   waddell  reed advisors small cap fund  \n",
       "3  esco tore srl                                                         \n",
       "4  russell total return fund  ltd                                        "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check top occuring words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "count_vectorizer = CountVectorizer(analyzer='word', ngram_range=(1, 1), min_df=1)\n",
    "# X matrix where the row represents sentences and column is our one-hot vector for each token in our vocabulary\n",
    "X = count_vectorizer.fit_transform(companies[\"clean_name\"])\n",
    "\n",
    "# Vocabulary\n",
    "vocab = list(count_vectorizer.get_feature_names())\n",
    "\n",
    "# Column-wise sum of the X matrix.\n",
    "# It's some crazy numpy syntax that looks horribly unpythonic\n",
    "# For details, see http://stackoverflow.com/questions/3337301/numpy-matrix-to-array\n",
    "# and http://stackoverflow.com/questions/13567345/how-to-calculate-the-sum-of-all-columns-of-a-2d-numpy-array-efficiently\n",
    "counts = X.sum(axis=0).A1\n",
    "\n",
    "freq_distribution = Counter(dict(zip(vocab, counts)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'llc', 4877), (u'fund', 4281), (u'limited', 3836), (u'gmbh', 3293), (u'inc', 2717), (u'bv', 2320), (u'srl', 2307), (u'trust', 2225), (u'ltd', 1795), (u'sa', 1675), (u'co', 1445), (u'lp', 1338), (u'global', 1237), (u'de', 1146), (u'kg', 1135), (u'holding', 1124), (u'the', 1077), (u'company', 1071), (u'funds', 1068), (u'ab', 1014), (u'investment', 894), (u'sl', 831), (u'capital', 803), (u'international', 782), (u'of', 758), (u'equity', 755), (u'spa', 753), (u'di', 749), (u'holdings', 742), (u'as', 692), (u'ii', 668), (u'group', 661), (u'sicav', 660), (u'management', 602), (u'investments', 600), (u'portfolio', 564), (u'aps', 562), (u'partners', 543), (u'bond', 521), (u'societa', 521), (u'invest', 490), (u'sro', 482), (u'and', 478), (u'oy', 461), (u'bank', 446), (u'master', 435), (u'corporation', 434), (u'nv', 419), (u'services', 417), (u'plc', 412), (u'income', 411), (u'ag', 383), (u'fonds', 381), (u'beheer', 380), (u'us', 377), (u'spoka', 375), (u'settlement', 368), (u'mbh', 361), (u'uk', 355), (u'europe', 355), (u'series', 351), (u'etf', 350), (u'asset', 345), (u'index', 342), (u'credit', 339), (u'in', 339), (u'sas', 335), (u'und', 330), (u'properties', 326), (u'sp', 323), (u'energy', 317), (u'street', 316), (u'snc', 309), (u'multi', 307), (u'finance', 303), (u'public', 276), (u'real', 272), (u'trading', 268), (u'aktiebolag', 264), (u'emerging', 257), (u'rl', 256), (u'solutions', 255), (u'markets', 255), (u'iii', 253), (u'van', 252), (u'tarsasag', 251), (u'korlatolt', 245), (u'felelossegu', 245), (u'growth', 244), (u'for', 242), (u'pty', 240), (u'oo', 239), (u'pension', 238), (u'stiftung', 237), (u'opportunities', 234), (u'estate', 233), (u'partnership', 233), (u'odpowiedzialnoscia', 231), (u'ograniczona', 231), (u'property', 230)]\n"
     ]
    }
   ],
   "source": [
    "print (freq_distribution.most_common(100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use TfidfVectorizer to generate company name vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer( max_df=0.8, ngram_range=(2,3), analyzer=\"char\",\n",
    "                                       token_pattern='(\\S+)', )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "company_vectors = tfidf_vectorizer.fit_transform(companies[\"clean_name\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(company_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 15375)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "company_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company_id</th>\n",
       "      <th>name</th>\n",
       "      <th>clean_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>243419</td>\n",
       "      <td>State Street Bank and Trust Company World Index Common Trust Funds - State Street 1-3 Year U.S. Agency Index Non-Lending Common Trust Fund</td>\n",
       "      <td>state street bank and trust company world index common trust funds   state street 1 3 year us agency index non lending common trust fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>272569</td>\n",
       "      <td>Autobedrijf Vander Stichele NV</td>\n",
       "      <td>autobedrijf vander stichele nv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>17447</td>\n",
       "      <td>INVESTEC FUNDS SERIES II - AMERICAN FUND</td>\n",
       "      <td>investec funds series ii   american fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>431383</td>\n",
       "      <td>GLG Technology Fund</td>\n",
       "      <td>glg technology fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>240428</td>\n",
       "      <td>MERSEYSIDE ESTATES LIMITED</td>\n",
       "      <td>merseyside estates limited</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       company_id  \\\n",
       "49995  243419       \n",
       "49996  272569       \n",
       "49997  17447        \n",
       "49998  431383       \n",
       "49999  240428       \n",
       "\n",
       "                                                                                                                                             name  \\\n",
       "49995  State Street Bank and Trust Company World Index Common Trust Funds - State Street 1-3 Year U.S. Agency Index Non-Lending Common Trust Fund   \n",
       "49996  Autobedrijf Vander Stichele NV                                                                                                               \n",
       "49997  INVESTEC FUNDS SERIES II - AMERICAN FUND                                                                                                     \n",
       "49998  GLG Technology Fund                                                                                                                          \n",
       "49999  MERSEYSIDE ESTATES LIMITED                                                                                                                   \n",
       "\n",
       "                                                                                                                                     clean_name  \n",
       "49995  state street bank and trust company world index common trust funds   state street 1 3 year us agency index non lending common trust fund  \n",
       "49996  autobedrijf vander stichele nv                                                                                                            \n",
       "49997  investec funds series ii   american fund                                                                                                  \n",
       "49998  glg technology fund                                                                                                                       \n",
       "49999  merseyside estates limited                                                                                                                "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "companies.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15375,)\n"
     ]
    }
   ],
   "source": [
    "type(company_vectors[0].todense())\n",
    "print np.squeeze(np.asarray(company_vectors[0].todense())).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Indexing using Annoy.\n",
    "Approximate nearest neigbour using random projections. Indexes once built are static files and can be used across processes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make map of company_ids to Annoy_index_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annoy_ids are from 0 to len(company_ids) -1 \n",
    "# annoy_ids are stored in list. List index are annoy_ids and the value is teh correspodnign company_id\n",
    "# Company_ids are stored in a dict. The key is company id and the value is annoy_id\n",
    "def build_company_annoy_maps(company_ids):\n",
    "    annoy2company = []\n",
    "    company2annoy = defaultdict(lambda:0)\n",
    "    for c_id in company_ids:\n",
    "        if c_id not in company2annoy:\n",
    "            annoy2company.append(c_id)\n",
    "            company2annoy[c_id] = len(annoy2company)-1\n",
    "    return annoy2company, company2annoy  \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "annoy2company, company2annoy = build_company_annoy_maps(companies[\"company_id\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000\n",
      "50000\n"
     ]
    }
   ],
   "source": [
    "print len(annoy2company)\n",
    "print len(company2annoy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34100 482478 34100\n",
      "4580 73643 4580\n",
      "30891 185217 30891\n",
      "42096 185101 42096\n",
      "41728 188097 41728\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "for i in range(5):\n",
    "    annoy_id = random.randint(0,50000)\n",
    "    assert annoy_id ==  company2annoy[annoy2company[annoy_id]]\n",
    "    print annoy_id, annoy2company[annoy_id], company2annoy[annoy2company[annoy_id]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build annoy Index\n",
    "Uncoment below code to build the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index_size = len(tfidf_vectorizer.vocabulary_)\n",
    "# table = AnnoyIndex(index_size)\n",
    "\n",
    "# for i in range(company_vectors.shape[0]):\n",
    "#     if i%10000==0:\n",
    "#         print \"indexed %s items\"%i\n",
    "#     table.add_item(i,np.squeeze(np.asarray(company_vectors[i].todense())))\n",
    "    \n",
    "# table.build(100)\n",
    "# table.save(file_path+\"annoy_index.ann\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load index and search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_size = len(tfidf_vectorizer.vocabulary_)\n",
    "table = AnnoyIndex(index_size)\n",
    "table.load(file_path+\"annoy_index.ann\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find neighbours of train set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create training name dense vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.merge(companies, how=\"left\", on=\"company_id\", suffixes=('','_truth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_nearest_neighbours(vectors, num_neighbours=1, search_nodes=-1, include_distances=True):\n",
    "    neighbours = np.empty(shape=(len(vectors),num_neighbours), dtype=np.int32)\n",
    "    distances = np.empty(shape=(len(vectors),num_neighbours))\n",
    "#     print len(vectors)\n",
    "#     print neighbours.shape\n",
    "#     print distances.shape\n",
    "    for idx,v in enumerate(vectors):\n",
    "        annoy_ids, annoy_distances = table.get_nns_by_vector(v, n=num_neighbours, search_k=search_nodes, include_distances=include_distances)\n",
    "        neighbours[idx,:] = annoy_ids\n",
    "        distances[idx,:] = annoy_distances\n",
    "    return neighbours, distances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Since we need to predict ground truth we set prediction as the nearest neighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9min 51s, sys: 543 ms, total: 9min 51s\n",
      "Wall time: 9min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_vectors = tfidf_vectorizer.transform(train[\"clean_name\"])\n",
    "train_vectors_dense = np.squeeze(np.asarray(train_vectors.todense()))\n",
    "neighbours, distances = get_nearest_neighbours(train_vectors_dense)\n",
    "neighbour_company_ids= np.vectorize(lambda x: annoy2company[x])(neighbours)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11069, 15375)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([438615, 526042, 281405, ...,  26071, 305713, 471477])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_1 = neighbour_company_ids[:,0]\n",
    "rank_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"predicted_id\"] = rank_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "train= train.merge(companies[['company_id', 'name']], left_on=\"predicted_id\", \n",
    "                   right_on=\"company_id\", suffixes=(\"\",\"_predicted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_index</th>\n",
       "      <th>name</th>\n",
       "      <th>company_id</th>\n",
       "      <th>predicted_id</th>\n",
       "      <th>name_predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>BERK ELEY LLP</td>\n",
       "      <td>438615</td>\n",
       "      <td>438615</td>\n",
       "      <td>BERKELEY PARTNERS LLP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>RBPA Leeuw</td>\n",
       "      <td>526042</td>\n",
       "      <td>526042</td>\n",
       "      <td>RBPA Leeuw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19</td>\n",
       "      <td>Waddell &amp; Reed Advisors Funds - Waddell &amp; Reed Advisors Small Cap Fund</td>\n",
       "      <td>281405</td>\n",
       "      <td>281405</td>\n",
       "      <td>Waddell &amp; Reed Advisors Funds - Waddell &amp; Reed Advisors Small Cap Fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>ESCO TORE S.R.L.</td>\n",
       "      <td>79550</td>\n",
       "      <td>79550</td>\n",
       "      <td>ESCO TORRE S.R.L.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>Russell Total Return Fund.) Ltd.</td>\n",
       "      <td>351412</td>\n",
       "      <td>351412</td>\n",
       "      <td>Russell Total Return Fund (Quarterly) Ltd.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>83</td>\n",
       "      <td>Cornel Global B.V.</td>\n",
       "      <td>370746</td>\n",
       "      <td>370746</td>\n",
       "      <td>Cornel B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>57000</td>\n",
       "      <td>Cornel Inves. B.V.</td>\n",
       "      <td>370746</td>\n",
       "      <td>370746</td>\n",
       "      <td>Cornel B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>123</td>\n",
       "      <td>Tokio Marine iLfe Insurance Singapore Ltd.</td>\n",
       "      <td>285532</td>\n",
       "      <td>285532</td>\n",
       "      <td>Tokio Marine Life Insurance Singapore Ltd.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>141</td>\n",
       "      <td>RIjSTO RANTE LO DI DI SPV E C. - S.A.S.</td>\n",
       "      <td>211110</td>\n",
       "      <td>211110</td>\n",
       "      <td>RISTORANTE LO STRETTOIO DI PEDRO DI VITO E C. - S.A.S.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>22804</td>\n",
       "      <td>RISTORANTE FACEC OOK DI HU C. SNC</td>\n",
       "      <td>-1</td>\n",
       "      <td>211110</td>\n",
       "      <td>RISTORANTE LO STRETTOIO DI PEDRO DI VITO E C. - S.A.S.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>157</td>\n",
       "      <td>Cham äleonreisen GmbH</td>\n",
       "      <td>503871</td>\n",
       "      <td>503871</td>\n",
       "      <td>Chamäleon Reisen GmbH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>158</td>\n",
       "      <td>SAGA SHIPHOLDINGnorway SICAV) AS</td>\n",
       "      <td>450734</td>\n",
       "      <td>450734</td>\n",
       "      <td>SAGA SHIPHOLDING (NORWAY) AS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>167</td>\n",
       "      <td>EAGLE FUND</td>\n",
       "      <td>424502</td>\n",
       "      <td>424502</td>\n",
       "      <td>EAGLE FUND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>176</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "      <td>634022</td>\n",
       "      <td>634022</td>\n",
       "      <td>PRIMCOM SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>181</td>\n",
       "      <td>Snap-on Logistics Company</td>\n",
       "      <td>117137</td>\n",
       "      <td>117137</td>\n",
       "      <td>Snap-on Logistics Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>24105</td>\n",
       "      <td>Company-on Logi. nap</td>\n",
       "      <td>117137</td>\n",
       "      <td>117137</td>\n",
       "      <td>Snap-on Logistics Company</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>198</td>\n",
       "      <td>Norstar Integrityltd Holdings</td>\n",
       "      <td>125003</td>\n",
       "      <td>315716</td>\n",
       "      <td>North Star Investment Holdings LLC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>205</td>\n",
       "      <td>Blabc MI B.V.</td>\n",
       "      <td>495752</td>\n",
       "      <td>442158</td>\n",
       "      <td>C-Mill B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>65628</td>\n",
       "      <td>C-Miol BV</td>\n",
       "      <td>442158</td>\n",
       "      <td>442158</td>\n",
       "      <td>C-Mill B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>237</td>\n",
       "      <td>M &amp; Z Associates Inves tment I LLC</td>\n",
       "      <td>276628</td>\n",
       "      <td>276628</td>\n",
       "      <td>M &amp; Z Associates I LLC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>245</td>\n",
       "      <td>K&amp;H származtatott LIMITED innovatív Amerika tőkevédett zárt végű alap</td>\n",
       "      <td>636832</td>\n",
       "      <td>412027</td>\n",
       "      <td>CIB Aranytartalék 2 Tokevédett Származtatott Alap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>267</td>\n",
       "      <td>Thomas H Lee Equity VI Access EMP LP</td>\n",
       "      <td>238574</td>\n",
       "      <td>238574</td>\n",
       "      <td>Thomas H Lee Equity VI Access EMP LP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>64706</td>\n",
       "      <td>Equity H Lee Thomas VI Access EMP Holdi. LP</td>\n",
       "      <td>238574</td>\n",
       "      <td>238574</td>\n",
       "      <td>Thomas H Lee Equity VI Access EMP LP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>271</td>\n",
       "      <td>Yello wflashpartners Captal LP</td>\n",
       "      <td>385359</td>\n",
       "      <td>385359</td>\n",
       "      <td>Yellowflash Capital Partners LP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>296</td>\n",
       "      <td>ASSMANN Distribution Trsut Sp. z oo</td>\n",
       "      <td>367282</td>\n",
       "      <td>367282</td>\n",
       "      <td>ASSMANN Distribution Sp. z o.o.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>326</td>\n",
       "      <td>ALMI LN</td>\n",
       "      <td>72969</td>\n",
       "      <td>72969</td>\n",
       "      <td>ALMI NAVIGATION LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>329</td>\n",
       "      <td>SOPRA. STERIA</td>\n",
       "      <td>378164</td>\n",
       "      <td>378164</td>\n",
       "      <td>SOPRA STERIA GROUP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>332</td>\n",
       "      <td>vPE III VG B.V.</td>\n",
       "      <td>505465</td>\n",
       "      <td>505465</td>\n",
       "      <td>SPE III Van Gogh B.V.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>342</td>\n",
       "      <td>Cinel ande et Inc.</td>\n",
       "      <td>238415</td>\n",
       "      <td>238415</td>\n",
       "      <td>Cinelande et Associes Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>361</td>\n",
       "      <td>Morgan Stanleyindia Services Lymited</td>\n",
       "      <td>377777</td>\n",
       "      <td>377777</td>\n",
       "      <td>Morgan Stanley India Services Private Limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11039</th>\n",
       "      <td>8258</td>\n",
       "      <td>OFFSH. SUPPORT VESSELS V LIMITED</td>\n",
       "      <td>-1</td>\n",
       "      <td>289993</td>\n",
       "      <td>OFFSHORE SUPPORT VESSELS VII LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11040</th>\n",
       "      <td>44017</td>\n",
       "      <td>Fairfield Belmodt L.P.</td>\n",
       "      <td>-1</td>\n",
       "      <td>345464</td>\n",
       "      <td>Fairfield Bristol, L.P.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11041</th>\n",
       "      <td>83776</td>\n",
       "      <td>CENTRO ODON. STUDI RIUNITI S.R.L. SIGLABILEcosmer S.R.L.</td>\n",
       "      <td>-1</td>\n",
       "      <td>205714</td>\n",
       "      <td>STUDI ECONOMICI E PROGETTI INTEGRATI SPA SIGLABILE \"SEPI SPA\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11042</th>\n",
       "      <td>20203</td>\n",
       "      <td>FFA Filmförderungsanstalt</td>\n",
       "      <td>-1</td>\n",
       "      <td>18692</td>\n",
       "      <td>Erste Abwicklungsanstalt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11043</th>\n",
       "      <td>5782</td>\n",
       "      <td>Lord Abbett Investment Trust - Lord Abbett Yield High LIMITED Fund</td>\n",
       "      <td>-1</td>\n",
       "      <td>391582</td>\n",
       "      <td>Lord Abbett Investment Trust - Lord Abbett Core Fixed Income Fund</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11044</th>\n",
       "      <td>9523</td>\n",
       "      <td>Stichting Aandelen Administratiekantoorprimapa\"</td>\n",
       "      <td>-1</td>\n",
       "      <td>573680</td>\n",
       "      <td>Stichting Administratiekantoor GVW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11045</th>\n",
       "      <td>1920</td>\n",
       "      <td>POWERCAT S.A.S. DI CATALANO LUISA &amp; C.</td>\n",
       "      <td>-1</td>\n",
       "      <td>208296</td>\n",
       "      <td>CATALANO MARINA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11046</th>\n",
       "      <td>9994</td>\n",
       "      <td>COM Agroindustrial Corp. Limited</td>\n",
       "      <td>-1</td>\n",
       "      <td>532785</td>\n",
       "      <td>D.B. CORP LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11047</th>\n",
       "      <td>67181</td>\n",
       "      <td>DBF SRL</td>\n",
       "      <td>-1</td>\n",
       "      <td>14110</td>\n",
       "      <td>C. &amp; F. S.R.L.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11048</th>\n",
       "      <td>46429</td>\n",
       "      <td>Simpeleen Irnc Osuuspankki</td>\n",
       "      <td>-1</td>\n",
       "      <td>361252</td>\n",
       "      <td>Keiteleen Osuuspankki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11049</th>\n",
       "      <td>72330</td>\n",
       "      <td>DATF</td>\n",
       "      <td>-1</td>\n",
       "      <td>223147</td>\n",
       "      <td>INDATECH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11050</th>\n",
       "      <td>60737</td>\n",
       "      <td>FOURNITURE-LA PC</td>\n",
       "      <td>-1</td>\n",
       "      <td>625742</td>\n",
       "      <td>SATURNIA S.R.L.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11051</th>\n",
       "      <td>3264</td>\n",
       "      <td>BIG SURstftung</td>\n",
       "      <td>-1</td>\n",
       "      <td>402742</td>\n",
       "      <td>Reinhard-Süring-Stiftung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11052</th>\n",
       "      <td>84907</td>\n",
       "      <td>Inc nAalyticshost Inve stment.</td>\n",
       "      <td>-1</td>\n",
       "      <td>584739</td>\n",
       "      <td>analyticon instruments gmbh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11053</th>\n",
       "      <td>17646</td>\n",
       "      <td>Collier Center Inc PT, LLC</td>\n",
       "      <td>-1</td>\n",
       "      <td>438007</td>\n",
       "      <td>HASC Center, Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11054</th>\n",
       "      <td>73393</td>\n",
       "      <td>Deuts chland LEAD GmbH</td>\n",
       "      <td>-1</td>\n",
       "      <td>245657</td>\n",
       "      <td>Fortis Lease Deutschland GmbH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11055</th>\n",
       "      <td>96774</td>\n",
       "      <td>BULLEX JSC</td>\n",
       "      <td>-1</td>\n",
       "      <td>530681</td>\n",
       "      <td>BUDLEX S.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11056</th>\n",
       "      <td>83876</td>\n",
       "      <td>Recycling Limited</td>\n",
       "      <td>-1</td>\n",
       "      <td>218123</td>\n",
       "      <td>VALPAK RECYCLING LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11057</th>\n",
       "      <td>98603</td>\n",
       "      <td>TERMO IDRAULICA BONFANTI A. S.R.L.</td>\n",
       "      <td>-1</td>\n",
       "      <td>314336</td>\n",
       "      <td>STANDARD HIDRAULICA SA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11058</th>\n",
       "      <td>18904</td>\n",
       "      <td>Enterprises Moody Ptyltd</td>\n",
       "      <td>-1</td>\n",
       "      <td>193158</td>\n",
       "      <td>Woody Paylor Enterprises Ltd.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11059</th>\n",
       "      <td>54001</td>\n",
       "      <td>cNUOVA CC S.P.A.</td>\n",
       "      <td>-1</td>\n",
       "      <td>513450</td>\n",
       "      <td>Banca Nuova S.p.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11060</th>\n",
       "      <td>52047</td>\n",
       "      <td>Synapsia SA</td>\n",
       "      <td>-1</td>\n",
       "      <td>40022</td>\n",
       "      <td>SYNAPCELL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11061</th>\n",
       "      <td>64711</td>\n",
       "      <td>INVEST MONE TAIRE 3</td>\n",
       "      <td>-1</td>\n",
       "      <td>623235</td>\n",
       "      <td>INVEST MONETAIRE 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11062</th>\n",
       "      <td>1788</td>\n",
       "      <td>AHL Man Diversified PCC Lmited - AHL Man Diversified (Guernsey) Class GmbH D JPY Shares</td>\n",
       "      <td>-1</td>\n",
       "      <td>228906</td>\n",
       "      <td>AHL Strategies PCC Limited - Man AHL Diversified 2XL Shares</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11063</th>\n",
       "      <td>64354</td>\n",
       "      <td>AMIK ITALIA S.P.A.</td>\n",
       "      <td>-1</td>\n",
       "      <td>396406</td>\n",
       "      <td>OLIP ITALIA S.P.A.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11064</th>\n",
       "      <td>70429</td>\n",
       "      <td>Toyota Auto Rece. 2013-A Owner Trust</td>\n",
       "      <td>-1</td>\n",
       "      <td>112611</td>\n",
       "      <td>First Investors Auto Owner Trust 2013-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11065</th>\n",
       "      <td>76317</td>\n",
       "      <td>KD GmcH Inve stment</td>\n",
       "      <td>-1</td>\n",
       "      <td>305884</td>\n",
       "      <td>RKD Investment Holdings Limited</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11066</th>\n",
       "      <td>66299</td>\n",
       "      <td>GxRDM Cuatro, LP</td>\n",
       "      <td>-1</td>\n",
       "      <td>14680</td>\n",
       "      <td>CUATROGE SL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11067</th>\n",
       "      <td>34451</td>\n",
       "      <td>BP Explo. (Dleta) Limited</td>\n",
       "      <td>-1</td>\n",
       "      <td>305713</td>\n",
       "      <td>BOREALIS EXPLORATION LIMITED</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11068</th>\n",
       "      <td>31145</td>\n",
       "      <td>Isar Valley Capital GmiH</td>\n",
       "      <td>-1</td>\n",
       "      <td>471477</td>\n",
       "      <td>Sugar Valley Inc.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11069 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       train_index  \\\n",
       "0      5             \n",
       "1      9             \n",
       "2      19            \n",
       "3      22            \n",
       "4      72            \n",
       "5      83            \n",
       "6      57000         \n",
       "7      123           \n",
       "8      141           \n",
       "9      22804         \n",
       "10     157           \n",
       "11     158           \n",
       "12     167           \n",
       "13     176           \n",
       "14     181           \n",
       "15     24105         \n",
       "16     198           \n",
       "17     205           \n",
       "18     65628         \n",
       "19     237           \n",
       "20     245           \n",
       "21     267           \n",
       "22     64706         \n",
       "23     271           \n",
       "24     296           \n",
       "25     326           \n",
       "26     329           \n",
       "27     332           \n",
       "28     342           \n",
       "29     361           \n",
       "...    ...           \n",
       "11039  8258          \n",
       "11040  44017         \n",
       "11041  83776         \n",
       "11042  20203         \n",
       "11043  5782          \n",
       "11044  9523          \n",
       "11045  1920          \n",
       "11046  9994          \n",
       "11047  67181         \n",
       "11048  46429         \n",
       "11049  72330         \n",
       "11050  60737         \n",
       "11051  3264          \n",
       "11052  84907         \n",
       "11053  17646         \n",
       "11054  73393         \n",
       "11055  96774         \n",
       "11056  83876         \n",
       "11057  98603         \n",
       "11058  18904         \n",
       "11059  54001         \n",
       "11060  52047         \n",
       "11061  64711         \n",
       "11062  1788          \n",
       "11063  64354         \n",
       "11064  70429         \n",
       "11065  76317         \n",
       "11066  66299         \n",
       "11067  34451         \n",
       "11068  31145         \n",
       "\n",
       "                                                                                          name  \\\n",
       "0      BERK ELEY LLP                                                                             \n",
       "1      RBPA Leeuw                                                                                \n",
       "2      Waddell & Reed Advisors Funds - Waddell & Reed Advisors Small Cap Fund                    \n",
       "3      ESCO TORE S.R.L.                                                                          \n",
       "4      Russell Total Return Fund.) Ltd.                                                          \n",
       "5      Cornel Global B.V.                                                                        \n",
       "6      Cornel Inves. B.V.                                                                        \n",
       "7      Tokio Marine iLfe Insurance Singapore Ltd.                                                \n",
       "8      RIjSTO RANTE LO DI DI SPV E C. - S.A.S.                                                   \n",
       "9      RISTORANTE FACEC OOK DI HU C. SNC                                                         \n",
       "10     Cham äleonreisen GmbH                                                                     \n",
       "11     SAGA SHIPHOLDINGnorway SICAV) AS                                                          \n",
       "12     EAGLE FUND                                                                                \n",
       "13     PRIMCOM SA                                                                                \n",
       "14     Snap-on Logistics Company                                                                 \n",
       "15     Company-on Logi. nap                                                                      \n",
       "16     Norstar Integrityltd Holdings                                                             \n",
       "17     Blabc MI B.V.                                                                             \n",
       "18     C-Miol BV                                                                                 \n",
       "19     M & Z Associates Inves tment I LLC                                                        \n",
       "20     K&H származtatott LIMITED innovatív Amerika tőkevédett zárt végű alap                     \n",
       "21     Thomas H Lee Equity VI Access EMP LP                                                      \n",
       "22     Equity H Lee Thomas VI Access EMP Holdi. LP                                               \n",
       "23     Yello wflashpartners Captal LP                                                            \n",
       "24     ASSMANN Distribution Trsut Sp. z oo                                                       \n",
       "25     ALMI LN                                                                                   \n",
       "26     SOPRA. STERIA                                                                             \n",
       "27     vPE III VG B.V.                                                                           \n",
       "28     Cinel ande et Inc.                                                                        \n",
       "29     Morgan Stanleyindia Services Lymited                                                      \n",
       "...                                     ...                                                      \n",
       "11039  OFFSH. SUPPORT VESSELS V LIMITED                                                          \n",
       "11040  Fairfield Belmodt L.P.                                                                    \n",
       "11041  CENTRO ODON. STUDI RIUNITI S.R.L. SIGLABILEcosmer S.R.L.                                  \n",
       "11042  FFA Filmförderungsanstalt                                                                 \n",
       "11043  Lord Abbett Investment Trust - Lord Abbett Yield High LIMITED Fund                        \n",
       "11044  Stichting Aandelen Administratiekantoorprimapa\"                                           \n",
       "11045  POWERCAT S.A.S. DI CATALANO LUISA & C.                                                    \n",
       "11046  COM Agroindustrial Corp. Limited                                                          \n",
       "11047  DBF SRL                                                                                   \n",
       "11048  Simpeleen Irnc Osuuspankki                                                                \n",
       "11049  DATF                                                                                      \n",
       "11050  FOURNITURE-LA PC                                                                          \n",
       "11051  BIG SURstftung                                                                            \n",
       "11052  Inc nAalyticshost Inve stment.                                                            \n",
       "11053  Collier Center Inc PT, LLC                                                                \n",
       "11054  Deuts chland LEAD GmbH                                                                    \n",
       "11055  BULLEX JSC                                                                                \n",
       "11056  Recycling Limited                                                                         \n",
       "11057  TERMO IDRAULICA BONFANTI A. S.R.L.                                                        \n",
       "11058  Enterprises Moody Ptyltd                                                                  \n",
       "11059  cNUOVA CC S.P.A.                                                                          \n",
       "11060  Synapsia SA                                                                               \n",
       "11061  INVEST MONE TAIRE 3                                                                       \n",
       "11062  AHL Man Diversified PCC Lmited - AHL Man Diversified (Guernsey) Class GmbH D JPY Shares   \n",
       "11063  AMIK ITALIA S.P.A.                                                                        \n",
       "11064  Toyota Auto Rece. 2013-A Owner Trust                                                      \n",
       "11065  KD GmcH Inve stment                                                                       \n",
       "11066  GxRDM Cuatro, LP                                                                          \n",
       "11067  BP Explo. (Dleta) Limited                                                                 \n",
       "11068  Isar Valley Capital GmiH                                                                  \n",
       "\n",
       "       company_id  predicted_id  \\\n",
       "0      438615      438615         \n",
       "1      526042      526042         \n",
       "2      281405      281405         \n",
       "3      79550       79550          \n",
       "4      351412      351412         \n",
       "5      370746      370746         \n",
       "6      370746      370746         \n",
       "7      285532      285532         \n",
       "8      211110      211110         \n",
       "9     -1           211110         \n",
       "10     503871      503871         \n",
       "11     450734      450734         \n",
       "12     424502      424502         \n",
       "13     634022      634022         \n",
       "14     117137      117137         \n",
       "15     117137      117137         \n",
       "16     125003      315716         \n",
       "17     495752      442158         \n",
       "18     442158      442158         \n",
       "19     276628      276628         \n",
       "20     636832      412027         \n",
       "21     238574      238574         \n",
       "22     238574      238574         \n",
       "23     385359      385359         \n",
       "24     367282      367282         \n",
       "25     72969       72969          \n",
       "26     378164      378164         \n",
       "27     505465      505465         \n",
       "28     238415      238415         \n",
       "29     377777      377777         \n",
       "...       ...         ...         \n",
       "11039 -1           289993         \n",
       "11040 -1           345464         \n",
       "11041 -1           205714         \n",
       "11042 -1           18692          \n",
       "11043 -1           391582         \n",
       "11044 -1           573680         \n",
       "11045 -1           208296         \n",
       "11046 -1           532785         \n",
       "11047 -1           14110          \n",
       "11048 -1           361252         \n",
       "11049 -1           223147         \n",
       "11050 -1           625742         \n",
       "11051 -1           402742         \n",
       "11052 -1           584739         \n",
       "11053 -1           438007         \n",
       "11054 -1           245657         \n",
       "11055 -1           530681         \n",
       "11056 -1           218123         \n",
       "11057 -1           314336         \n",
       "11058 -1           193158         \n",
       "11059 -1           513450         \n",
       "11060 -1           40022          \n",
       "11061 -1           623235         \n",
       "11062 -1           228906         \n",
       "11063 -1           396406         \n",
       "11064 -1           112611         \n",
       "11065 -1           305884         \n",
       "11066 -1           14680          \n",
       "11067 -1           305713         \n",
       "11068 -1           471477         \n",
       "\n",
       "                                                               name_predicted  \n",
       "0      BERKELEY PARTNERS LLP                                                   \n",
       "1      RBPA Leeuw                                                              \n",
       "2      Waddell & Reed Advisors Funds - Waddell & Reed Advisors Small Cap Fund  \n",
       "3      ESCO TORRE S.R.L.                                                       \n",
       "4      Russell Total Return Fund (Quarterly) Ltd.                              \n",
       "5      Cornel B.V.                                                             \n",
       "6      Cornel B.V.                                                             \n",
       "7      Tokio Marine Life Insurance Singapore Ltd.                              \n",
       "8      RISTORANTE LO STRETTOIO DI PEDRO DI VITO E C. - S.A.S.                  \n",
       "9      RISTORANTE LO STRETTOIO DI PEDRO DI VITO E C. - S.A.S.                  \n",
       "10     Chamäleon Reisen GmbH                                                   \n",
       "11     SAGA SHIPHOLDING (NORWAY) AS                                            \n",
       "12     EAGLE FUND                                                              \n",
       "13     PRIMCOM SA                                                              \n",
       "14     Snap-on Logistics Company                                               \n",
       "15     Snap-on Logistics Company                                               \n",
       "16     North Star Investment Holdings LLC                                      \n",
       "17     C-Mill B.V.                                                             \n",
       "18     C-Mill B.V.                                                             \n",
       "19     M & Z Associates I LLC                                                  \n",
       "20     CIB Aranytartalék 2 Tokevédett Származtatott Alap                       \n",
       "21     Thomas H Lee Equity VI Access EMP LP                                    \n",
       "22     Thomas H Lee Equity VI Access EMP LP                                    \n",
       "23     Yellowflash Capital Partners LP                                         \n",
       "24     ASSMANN Distribution Sp. z o.o.                                         \n",
       "25     ALMI NAVIGATION LIMITED                                                 \n",
       "26     SOPRA STERIA GROUP                                                      \n",
       "27     SPE III Van Gogh B.V.                                                   \n",
       "28     Cinelande et Associes Inc.                                              \n",
       "29     Morgan Stanley India Services Private Limited                           \n",
       "...                                              ...                           \n",
       "11039  OFFSHORE SUPPORT VESSELS VII LIMITED                                    \n",
       "11040  Fairfield Bristol, L.P.                                                 \n",
       "11041  STUDI ECONOMICI E PROGETTI INTEGRATI SPA SIGLABILE \"SEPI SPA\"           \n",
       "11042  Erste Abwicklungsanstalt                                                \n",
       "11043  Lord Abbett Investment Trust - Lord Abbett Core Fixed Income Fund       \n",
       "11044  Stichting Administratiekantoor GVW                                      \n",
       "11045  CATALANO MARINA                                                         \n",
       "11046  D.B. CORP LIMITED                                                       \n",
       "11047  C. & F. S.R.L.                                                          \n",
       "11048  Keiteleen Osuuspankki                                                   \n",
       "11049  INDATECH                                                                \n",
       "11050  SATURNIA S.R.L.                                                         \n",
       "11051  Reinhard-Süring-Stiftung                                                \n",
       "11052  analyticon instruments gmbh                                             \n",
       "11053  HASC Center, Inc.                                                       \n",
       "11054  Fortis Lease Deutschland GmbH                                           \n",
       "11055  BUDLEX S.A.                                                             \n",
       "11056  VALPAK RECYCLING LIMITED                                                \n",
       "11057  STANDARD HIDRAULICA SA                                                  \n",
       "11058  Woody Paylor Enterprises Ltd.                                           \n",
       "11059  Banca Nuova S.p.A.                                                      \n",
       "11060  SYNAPCELL                                                               \n",
       "11061  INVEST MONETAIRE 4                                                      \n",
       "11062  AHL Strategies PCC Limited - Man AHL Diversified 2XL Shares             \n",
       "11063  OLIP ITALIA S.P.A.                                                      \n",
       "11064  First Investors Auto Owner Trust 2013-2                                 \n",
       "11065  RKD Investment Holdings Limited                                         \n",
       "11066  CUATROGE SL                                                             \n",
       "11067  BOREALIS EXPLORATION LIMITED                                            \n",
       "11068  Sugar Valley Inc.                                                       \n",
       "\n",
       "[11069 rows x 5 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[[\"train_index\", \"name\", \"company_id\", \"predicted_id\", \"name_predicted\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 11069\n",
      "Accuracy | Correct predictions 0.551721\n",
      "Incorrect predictions 0.448279\n"
     ]
    }
   ],
   "source": [
    "total = train.shape[0]\n",
    "correct_predictions = train.loc[train[\"company_id\"]==train[\"predicted_id\"]].shape[0]\n",
    "incorrect_predictions = train.loc[train[\"company_id\"]!=train[\"predicted_id\"]].shape[0]\n",
    "print \"Total %d\"%total\n",
    "print \"Accuracy | Correct predictions %f\"% (float(correct_predictions)/total)\n",
    "print \"Incorrect predictions %f\"% (float(incorrect_predictions)/total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3361, 9)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[train[\"company_id\"]==-1].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accuracy on excluding -1 cases "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7922937208095485\n"
     ]
    }
   ],
   "source": [
    "train_sub = train.loc[train[\"company_id\"]!=-1]\n",
    "print np.sum(train_sub[\"company_id\"] == train_sub[\"predicted_id\"])/float(len(train_sub))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That is all!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}