02_Data_Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2942198c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import pytesseract\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "from glob import glob\n",
    "from tqdm import tqdm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a907a35",
   "metadata": {},
   "outputs": [],
   "source": [
    "imgPaths = glob('./Selected/*.jpeg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e024b3fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./Selected\\\\000.jpeg',\n",
       " './Selected\\\\002.jpeg',\n",
       " './Selected\\\\003.jpeg',\n",
       " './Selected\\\\004.jpeg',\n",
       " './Selected\\\\005.jpeg',\n",
       " './Selected\\\\006.jpeg',\n",
       " './Selected\\\\007.jpeg',\n",
       " './Selected\\\\008.jpeg',\n",
       " './Selected\\\\009.jpeg',\n",
       " './Selected\\\\010.jpeg',\n",
       " './Selected\\\\011.jpeg',\n",
       " './Selected\\\\012.jpeg',\n",
       " './Selected\\\\013.jpeg',\n",
       " './Selected\\\\014.jpeg',\n",
       " './Selected\\\\015.jpeg',\n",
       " './Selected\\\\016.jpeg',\n",
       " './Selected\\\\017.jpeg',\n",
       " './Selected\\\\018.jpeg',\n",
       " './Selected\\\\019.jpeg',\n",
       " './Selected\\\\020.jpeg',\n",
       " './Selected\\\\021.jpeg',\n",
       " './Selected\\\\022.jpeg',\n",
       " './Selected\\\\023.jpeg',\n",
       " './Selected\\\\024.jpeg',\n",
       " './Selected\\\\025.jpeg',\n",
       " './Selected\\\\026.jpeg',\n",
       " './Selected\\\\027.jpeg',\n",
       " './Selected\\\\028.jpeg',\n",
       " './Selected\\\\029.jpeg',\n",
       " './Selected\\\\030.jpeg',\n",
       " './Selected\\\\031.jpeg',\n",
       " './Selected\\\\032.jpeg',\n",
       " './Selected\\\\033.jpeg',\n",
       " './Selected\\\\034.jpeg',\n",
       " './Selected\\\\035.jpeg',\n",
       " './Selected\\\\036.jpeg',\n",
       " './Selected\\\\037.jpeg',\n",
       " './Selected\\\\038.jpeg',\n",
       " './Selected\\\\039.jpeg',\n",
       " './Selected\\\\040.jpeg',\n",
       " './Selected\\\\041.jpeg',\n",
       " './Selected\\\\042.jpeg',\n",
       " './Selected\\\\043.jpeg',\n",
       " './Selected\\\\044.jpeg',\n",
       " './Selected\\\\045.jpeg',\n",
       " './Selected\\\\046.jpeg',\n",
       " './Selected\\\\047.jpeg',\n",
       " './Selected\\\\048.jpeg',\n",
       " './Selected\\\\049.jpeg',\n",
       " './Selected\\\\050.jpeg',\n",
       " './Selected\\\\051.jpeg',\n",
       " './Selected\\\\052.jpeg',\n",
       " './Selected\\\\053.jpeg',\n",
       " './Selected\\\\054.jpeg',\n",
       " './Selected\\\\055.jpeg',\n",
       " './Selected\\\\056.jpeg',\n",
       " './Selected\\\\057.jpeg',\n",
       " './Selected\\\\058.jpeg',\n",
       " './Selected\\\\059.jpeg',\n",
       " './Selected\\\\060.jpeg',\n",
       " './Selected\\\\061.jpeg',\n",
       " './Selected\\\\062.jpeg',\n",
       " './Selected\\\\063.jpeg',\n",
       " './Selected\\\\064.jpeg',\n",
       " './Selected\\\\065.jpeg',\n",
       " './Selected\\\\066.jpeg',\n",
       " './Selected\\\\067.jpeg',\n",
       " './Selected\\\\068.jpeg',\n",
       " './Selected\\\\069.jpeg',\n",
       " './Selected\\\\070.jpeg',\n",
       " './Selected\\\\071.jpeg',\n",
       " './Selected\\\\072.jpeg',\n",
       " './Selected\\\\073.jpeg',\n",
       " './Selected\\\\074.jpeg',\n",
       " './Selected\\\\075.jpeg',\n",
       " './Selected\\\\076.jpeg',\n",
       " './Selected\\\\077.jpeg',\n",
       " './Selected\\\\078.jpeg',\n",
       " './Selected\\\\079.jpeg',\n",
       " './Selected\\\\080.jpeg',\n",
       " './Selected\\\\081.jpeg',\n",
       " './Selected\\\\082.jpeg',\n",
       " './Selected\\\\083.jpeg',\n",
       " './Selected\\\\084.jpeg',\n",
       " './Selected\\\\085.jpeg',\n",
       " './Selected\\\\086.jpeg',\n",
       " './Selected\\\\087.jpeg',\n",
       " './Selected\\\\088.jpeg',\n",
       " './Selected\\\\089.jpeg',\n",
       " './Selected\\\\090.jpeg',\n",
       " './Selected\\\\091.jpeg',\n",
       " './Selected\\\\092.jpeg',\n",
       " './Selected\\\\093.jpeg',\n",
       " './Selected\\\\094.jpeg',\n",
       " './Selected\\\\095.jpeg',\n",
       " './Selected\\\\096.jpeg',\n",
       " './Selected\\\\097.jpeg',\n",
       " './Selected\\\\098.jpeg',\n",
       " './Selected\\\\099.jpeg',\n",
       " './Selected\\\\100.jpeg',\n",
       " './Selected\\\\101.jpeg',\n",
       " './Selected\\\\102.jpeg',\n",
       " './Selected\\\\103.jpeg',\n",
       " './Selected\\\\104.jpeg',\n",
       " './Selected\\\\105.jpeg',\n",
       " './Selected\\\\106.jpeg',\n",
       " './Selected\\\\107.jpeg',\n",
       " './Selected\\\\108.jpeg',\n",
       " './Selected\\\\109.jpeg',\n",
       " './Selected\\\\110.jpeg',\n",
       " './Selected\\\\111.jpeg',\n",
       " './Selected\\\\112.jpeg',\n",
       " './Selected\\\\113.jpeg',\n",
       " './Selected\\\\114.jpeg',\n",
       " './Selected\\\\115.jpeg',\n",
       " './Selected\\\\116.jpeg',\n",
       " './Selected\\\\117.jpeg',\n",
       " './Selected\\\\118.jpeg',\n",
       " './Selected\\\\119.jpeg',\n",
       " './Selected\\\\120.jpeg',\n",
       " './Selected\\\\121.jpeg',\n",
       " './Selected\\\\122.jpeg',\n",
       " './Selected\\\\123.jpeg',\n",
       " './Selected\\\\124.jpeg',\n",
       " './Selected\\\\125.jpeg',\n",
       " './Selected\\\\126.jpeg',\n",
       " './Selected\\\\127.jpeg',\n",
       " './Selected\\\\128.jpeg',\n",
       " './Selected\\\\129.jpeg',\n",
       " './Selected\\\\130.jpeg',\n",
       " './Selected\\\\131.jpeg',\n",
       " './Selected\\\\132.jpeg',\n",
       " './Selected\\\\133.jpeg',\n",
       " './Selected\\\\134.jpeg',\n",
       " './Selected\\\\135.jpeg',\n",
       " './Selected\\\\136.jpeg',\n",
       " './Selected\\\\137.jpeg',\n",
       " './Selected\\\\138.jpeg',\n",
       " './Selected\\\\139.jpeg',\n",
       " './Selected\\\\140.jpeg',\n",
       " './Selected\\\\141.jpeg',\n",
       " './Selected\\\\142.jpeg',\n",
       " './Selected\\\\143.jpeg',\n",
       " './Selected\\\\144.jpeg',\n",
       " './Selected\\\\145.jpeg',\n",
       " './Selected\\\\146.jpeg',\n",
       " './Selected\\\\147.jpeg',\n",
       " './Selected\\\\148.jpeg',\n",
       " './Selected\\\\149.jpeg',\n",
       " './Selected\\\\150.jpeg',\n",
       " './Selected\\\\151.jpeg',\n",
       " './Selected\\\\152.jpeg',\n",
       " './Selected\\\\153.jpeg',\n",
       " './Selected\\\\154.jpeg',\n",
       " './Selected\\\\155.jpeg',\n",
       " './Selected\\\\156.jpeg',\n",
       " './Selected\\\\157.jpeg',\n",
       " './Selected\\\\158.jpeg',\n",
       " './Selected\\\\159.jpeg',\n",
       " './Selected\\\\160.jpeg',\n",
       " './Selected\\\\161.jpeg',\n",
       " './Selected\\\\162.jpeg',\n",
       " './Selected\\\\163.jpeg',\n",
       " './Selected\\\\164.jpeg',\n",
       " './Selected\\\\165.jpeg',\n",
       " './Selected\\\\166.jpeg',\n",
       " './Selected\\\\167.jpeg',\n",
       " './Selected\\\\168.jpeg',\n",
       " './Selected\\\\169.jpeg',\n",
       " './Selected\\\\170.jpeg',\n",
       " './Selected\\\\171.jpeg',\n",
       " './Selected\\\\172.jpeg',\n",
       " './Selected\\\\173.jpeg',\n",
       " './Selected\\\\174.jpeg',\n",
       " './Selected\\\\175.jpeg',\n",
       " './Selected\\\\176.jpeg',\n",
       " './Selected\\\\177.jpeg',\n",
       " './Selected\\\\178.jpeg',\n",
       " './Selected\\\\179.jpeg',\n",
       " './Selected\\\\180.jpeg',\n",
       " './Selected\\\\181.jpeg',\n",
       " './Selected\\\\182.jpeg',\n",
       " './Selected\\\\183.jpeg',\n",
       " './Selected\\\\184.jpeg',\n",
       " './Selected\\\\185.jpeg',\n",
       " './Selected\\\\186.jpeg',\n",
       " './Selected\\\\187.jpeg',\n",
       " './Selected\\\\188.jpeg',\n",
       " './Selected\\\\189.jpeg',\n",
       " './Selected\\\\190.jpeg',\n",
       " './Selected\\\\191.jpeg',\n",
       " './Selected\\\\192.jpeg',\n",
       " './Selected\\\\193.jpeg',\n",
       " './Selected\\\\194.jpeg',\n",
       " './Selected\\\\195.jpeg',\n",
       " './Selected\\\\196.jpeg',\n",
       " './Selected\\\\197.jpeg',\n",
       " './Selected\\\\198.jpeg',\n",
       " './Selected\\\\199.jpeg',\n",
       " './Selected\\\\200.jpeg',\n",
       " './Selected\\\\201.jpeg',\n",
       " './Selected\\\\202.jpeg',\n",
       " './Selected\\\\203.jpeg',\n",
       " './Selected\\\\204.jpeg',\n",
       " './Selected\\\\205.jpeg',\n",
       " './Selected\\\\206.jpeg',\n",
       " './Selected\\\\207.jpeg',\n",
       " './Selected\\\\208.jpeg',\n",
       " './Selected\\\\209.jpeg',\n",
       " './Selected\\\\210.jpeg',\n",
       " './Selected\\\\211.jpeg',\n",
       " './Selected\\\\212.jpeg',\n",
       " './Selected\\\\213.jpeg',\n",
       " './Selected\\\\214.jpeg',\n",
       " './Selected\\\\215.jpeg',\n",
       " './Selected\\\\216.jpeg',\n",
       " './Selected\\\\217.jpeg',\n",
       " './Selected\\\\218.jpeg',\n",
       " './Selected\\\\219.jpeg',\n",
       " './Selected\\\\220.jpeg',\n",
       " './Selected\\\\221.jpeg',\n",
       " './Selected\\\\222.jpeg',\n",
       " './Selected\\\\223.jpeg',\n",
       " './Selected\\\\224.jpeg',\n",
       " './Selected\\\\225.jpeg',\n",
       " './Selected\\\\226.jpeg',\n",
       " './Selected\\\\227.jpeg',\n",
       " './Selected\\\\228.jpeg',\n",
       " './Selected\\\\229.jpeg',\n",
       " './Selected\\\\230.jpeg',\n",
       " './Selected\\\\231.jpeg',\n",
       " './Selected\\\\232.jpeg',\n",
       " './Selected\\\\233.jpeg',\n",
       " './Selected\\\\234.jpeg',\n",
       " './Selected\\\\235.jpeg',\n",
       " './Selected\\\\236.jpeg',\n",
       " './Selected\\\\237.jpeg',\n",
       " './Selected\\\\238.jpeg',\n",
       " './Selected\\\\239.jpeg',\n",
       " './Selected\\\\240.jpeg',\n",
       " './Selected\\\\241.jpeg',\n",
       " './Selected\\\\242.jpeg',\n",
       " './Selected\\\\243.jpeg',\n",
       " './Selected\\\\244.jpeg',\n",
       " './Selected\\\\245.jpeg',\n",
       " './Selected\\\\246.jpeg',\n",
       " './Selected\\\\247.jpeg',\n",
       " './Selected\\\\248.jpeg',\n",
       " './Selected\\\\249.jpeg',\n",
       " './Selected\\\\250.jpeg',\n",
       " './Selected\\\\251.jpeg',\n",
       " './Selected\\\\252.jpeg',\n",
       " './Selected\\\\253.jpeg',\n",
       " './Selected\\\\254.jpeg',\n",
       " './Selected\\\\255.jpeg',\n",
       " './Selected\\\\256.jpeg',\n",
       " './Selected\\\\257.jpeg',\n",
       " './Selected\\\\258.jpeg',\n",
       " './Selected\\\\259.jpeg',\n",
       " './Selected\\\\260.jpeg',\n",
       " './Selected\\\\261.jpeg',\n",
       " './Selected\\\\262.jpeg',\n",
       " './Selected\\\\263.jpeg',\n",
       " './Selected\\\\264.jpeg',\n",
       " './Selected\\\\265.jpeg',\n",
       " './Selected\\\\266.jpeg',\n",
       " './Selected\\\\267.jpeg',\n",
       " './Selected\\\\268.jpeg',\n",
       " './Selected\\\\269.jpeg',\n",
       " './Selected\\\\270.jpeg',\n",
       " './Selected\\\\271.jpeg',\n",
       " './Selected\\\\272.jpeg',\n",
       " './Selected\\\\273.jpeg',\n",
       " './Selected\\\\274.jpeg',\n",
       " './Selected\\\\275.jpeg',\n",
       " './Selected\\\\276.jpeg',\n",
       " './Selected\\\\277.jpeg',\n",
       " './Selected\\\\278.jpeg',\n",
       " './Selected\\\\279.jpeg',\n",
       " './Selected\\\\280.jpeg',\n",
       " './Selected\\\\281.jpeg',\n",
       " './Selected\\\\282.jpeg',\n",
       " './Selected\\\\283.jpeg',\n",
       " './Selected\\\\284.jpeg',\n",
       " './Selected\\\\285.jpeg',\n",
       " './Selected\\\\286.jpeg',\n",
       " './Selected\\\\287.jpeg',\n",
       " './Selected\\\\288.jpeg',\n",
       " './Selected\\\\289.jpeg',\n",
       " './Selected\\\\290.jpeg',\n",
       " './Selected\\\\291.jpeg',\n",
       " './Selected\\\\292.jpeg',\n",
       " './Selected\\\\293.jpeg',\n",
       " './Selected\\\\294.jpeg',\n",
       " './Selected\\\\295.jpeg',\n",
       " './Selected\\\\296.jpeg',\n",
       " './Selected\\\\297.jpeg',\n",
       " './Selected\\\\298.jpeg',\n",
       " './Selected\\\\299.jpeg',\n",
       " './Selected\\\\300.jpeg']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgPaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3838c0d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Documents: 100%|█████████████████████████████████████████████████████████████████████████████████| 300/300 [06:13<00:00,  1.25s/it]\n"
     ]
    }
   ],
   "source": [
    "allDocuments = pd.DataFrame(columns=['id','text'])\n",
    "\n",
    "for imgPath in  tqdm(imgPaths,desc='Documents'):\n",
    "    \n",
    "    #imgPath = imgPaths[0]\n",
    "    _, filename = os.path.split(imgPath)\n",
    "    # extract data and text \n",
    "    image = cv2.imread(imgPath)\n",
    "    data = pytesseract.image_to_data(image)\n",
    "    dataList = list(map(lambda x: x.split('\\t'),data.split('\\n')))\n",
    "    df = pd.DataFrame(dataList[1:],columns=dataList[0])\n",
    "    df.dropna(inplace=True)\n",
    "    df['conf'] = df['conf'].astype(int)\n",
    "\n",
    "    useFulData = df.query('conf >= 30')\n",
    "\n",
    "    # Dataframe\n",
    "    Documents = pd.DataFrame()\n",
    "    Documents['text'] = useFulData['text']\n",
    "    Documents['id'] = filename\n",
    "    \n",
    "    # concatenation\n",
    "    allDocuments = pd.concat((allDocuments,Documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c229684c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>040-4852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>8881,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000.jpeg</td>\n",
       "      <td>90309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>300.jpeg</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>300.jpeg</td>\n",
       "      <td>DURBAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>300.jpeg</td>\n",
       "      <td>ROAD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>300.jpeg</td>\n",
       "      <td>DURBAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>300.jpeg</td>\n",
       "      <td>4001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10046 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id      text\n",
       "4   000.jpeg          \n",
       "8   000.jpeg         .\n",
       "9   000.jpeg  040-4852\n",
       "10  000.jpeg     8881,\n",
       "11  000.jpeg     90309\n",
       "..       ...       ...\n",
       "35  300.jpeg       169\n",
       "36  300.jpeg    DURBAN\n",
       "37  300.jpeg      ROAD\n",
       "39  300.jpeg    DURBAN\n",
       "41  300.jpeg      4001\n",
       "\n",
       "[10046 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allDocuments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "326b06f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "allDocuments.to_csv('documents.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5be6de3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}