Joblib querying of NMVW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0184f795",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f266f9bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from joblib import Parallel, delayed\n",
    "import time, math\n",
    "import pickle as pkl\n",
    "import re\n",
    "import json\n",
    "import csv\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import contextlib\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e01dc79a",
   "metadata": {},
   "source": [
    "# Read files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "56c47c1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('nmvw.json') as json_file: # json file of NMVW data\n",
    "    nmvw = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4280835c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('knowledge_graph_symbols_english_and_dutch.csv', index_col=0) # these are the symbols translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2a46a52d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>symbol</th>\n",
       "      <th>label</th>\n",
       "      <th>translation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aMGPDT</td>\n",
       "      <td>A.M.G.P.D.T.</td>\n",
       "      <td>AMGPDT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>broomSweeping</td>\n",
       "      <td>broom (sweeping)</td>\n",
       "      <td>bezem (vegen)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zither</td>\n",
       "      <td>zither</td>\n",
       "      <td>citer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>almsBowl</td>\n",
       "      <td>alms bowl</td>\n",
       "      <td>aalmoeskom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>almuce</td>\n",
       "      <td>almuce</td>\n",
       "      <td>almuce</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          symbol             label    translation\n",
       "0         aMGPDT      A.M.G.P.D.T.         AMGPDT\n",
       "1  broomSweeping  broom (sweeping)  bezem (vegen)\n",
       "2         zither            zither          citer\n",
       "3       almsBowl         alms bowl     aalmoeskom\n",
       "4         almuce            almuce         almuce"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=df.reset_index().drop(\"index\", axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75f99a3b",
   "metadata": {},
   "source": [
    "# Necessary Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "596abe30",
   "metadata": {},
   "outputs": [],
   "source": [
    "def querying(query_str):\n",
    "    # modifying a query term\n",
    "    clean_query_str = str(query_str).replace('*','').replace(')','').replace('(','').replace('&#39;',\"'\")\n",
    "        \n",
    "    match_dict = {}\n",
    "    \n",
    "    # iterate over NMVW\n",
    "    \n",
    "    for handle, values in nmvw.items():\n",
    "\n",
    "        found_in = ''\n",
    "\n",
    "        # searching in descriptions\n",
    "        if re.search(f\"\\\\b{clean_query_str}\\\\b\",values['desc'],re.IGNORECASE) != None:\n",
    "            found_in = 'desc'\n",
    "\n",
    "        if found_in == 'desc':\n",
    "            match_dict[handle] = {\"title\":values['title']}\n",
    "    \n",
    "    #query_results[query_str] = match_dict\n",
    "    return [query_str,match_dict]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b5bdea7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "@contextlib.contextmanager\n",
    "def tqdm_joblib(tqdm_object):\n",
    "    \"\"\"Context manager to patch joblib to report into tqdm progress bar given as argument\"\"\"\n",
    "    class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):\n",
    "        def __call__(self, *args, **kwargs):\n",
    "            tqdm_object.update(n=self.batch_size)\n",
    "            return super().__call__(*args, **kwargs)\n",
    "\n",
    "    old_batch_callback = joblib.parallel.BatchCompletionCallBack\n",
    "    joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback\n",
    "    try:\n",
    "        yield tqdm_object\n",
    "    finally:\n",
    "        joblib.parallel.BatchCompletionCallBack = old_batch_callback\n",
    "        tqdm_object.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bca62340",
   "metadata": {},
   "source": [
    "# Querying"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b5c8b2e5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "My calculation: 0it [00:00, ?it/s]/var/folders/kh/k3tdtq_s5197cmczt3hthfc40000gn/T/ipykernel_74841/1443666868.py:2: FutureWarning: The behavior of `series[i:j]` with an integer-dtype index is deprecated. In a future version, this will be treated as *label-based* indexing, consistent with e.g. `series[i]` lookups. To retain the old behavior, use `series.iloc[i:j]`. To get the future behavior, use `series.loc[i:j]`.\n",
      "  showMeResults = Parallel(n_jobs=16)(delayed(my_fun)(df.translation[i]) for i in range(len(df.translation[:10])))\n",
      "My calculation: 10it [00:09,  1.05it/s]\n"
     ]
    }
   ],
   "source": [
    "with tqdm_joblib(tqdm(desc=\"My calculation\")) as progress_bar:\n",
    "    showMeResults = Parallel(n_jobs=16)(delayed(querying)(df.translation[i]) for i in range(len(df.translation)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}