gpsProbeMatching_4m_folder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GPS Probe mapping for many .csv input files in  <raw_dir> within the date range <start_date>  and <end_date>. \n",
    "### The csv file names must indicate the dates.\n",
    "### For working with single input file use gpsProbeMatching.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from datetime import datetime ,date\n",
    "\n",
    "import sys,os\n",
    "sys.path.append('py/') \n",
    "\n",
    "import shutil\n",
    "from functions import initialize, get_points_within_target_region, df2gdf, plot_map, check_dir, check_dir, get_all_files_from_dir\n",
    "from preprocess import preprocess_data\n",
    "from map_matching import map_match_csv2gpx, map_match_csv2gpx_multithread\n",
    "from generate_route_by_pyroutelib import generate_osm_routes_main\n",
    "from config import max_threads, OUTPUT_DIR, INPUT_DIR, map_matched_gps_probe\n",
    "\n",
    "from get_raw_files import get_raw_files_main"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Input and Output locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "raw_dir = '/home/bidur/map_match_gps_data/raw_phl'\n",
    "mapmatching_input_dir = '/home/bidur/map_match_gps_data/raw_phl/phl_sample_clean/'\n",
    "backup_dir ='/home/bidur/map_match_gps_data/raw_phl/bak/'\n",
    "'''\n",
    "raw_dir = '/mnt/lv/heromiya/PHLMobilityData/'    ## original raw file location\n",
    "mapmatching_input_dir = '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/' ## prepare csv in requireds format(ap_id, ) for map-matching\n",
    "backup_dir = '/mnt/lv/bidur/PHL_raw_data/csv_4_mobmap/201907/' \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CSV prepare for input to  map-matching\n",
    "__Assumptions:__\n",
    "- start_date < end_date\n",
    "- raw input files are named with date in __yyyymmdd__ format. Ex. __NPL_data_20190404.csv__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- original csv files from *raw_dir* contains many fields.\n",
    "- Interesting fields in each csv are got via: \n",
    "         cut -d ',' -f 1,2,3,4,6,7,8,9 NPL_data_20190404.csv >  NPL_20190704.csv\n",
    "- timestamp field is creadted by joining different fileds using *py/get_raw_files.py*\n",
    "    - csv files with desired fields *(ap_id, timestamp, latitude, longitude)* are saved at : *mapmatching_input_dir*\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Files from __raw_dir__ are processed and saved in __mapmatching_input_dir__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selected Input Files: \n",
      "PDP_philippines_20190704.csv\n",
      "PDP_philippines_20190706.csv\n",
      "PDP_philippines_20190703.csv\n",
      "PDP_philippines_20190707.csv\n",
      "PDP_philippines_20190702.csv\n",
      "PDP_philippines_20190705.csv\n",
      "PDP_philippines_20190701.csv\n",
      "prepare_target_csv()\n",
      "\t\t Start:  PDP_philippines_20190704.csv 2020-11-19 14:29:01.671783\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190704.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190704.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190704.csv\n",
      "ap_id (all): 15852  ap_id (No JUMPS): 15791\n",
      "2020-11-19 14:47:59.227761,PDP_philippines_20190704.csv,15852,15852,15791,237229,212774,209902\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190704.csv 2020-11-19 14:47:59.241490\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190706.csv 2020-11-19 14:47:59.241615\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190706.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190706.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190706.csv\n",
      "ap_id (all): 16212  ap_id (No JUMPS): 16150\n",
      "2020-11-19 15:07:03.819639,PDP_philippines_20190706.csv,16212,16212,16150,236217,212808,209969\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190706.csv 2020-11-19 15:07:03.851959\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190703.csv 2020-11-19 15:07:03.852127\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190703.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190703.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190703.csv\n",
      "ap_id (all): 16128  ap_id (No JUMPS): 16071\n",
      "2020-11-19 15:25:57.650515,PDP_philippines_20190703.csv,16128,16128,16071,233101,209303,207098\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190703.csv 2020-11-19 15:25:57.670271\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190707.csv 2020-11-19 15:25:57.670368\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190707.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190707.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190707.csv\n",
      "ap_id (all): 15640  ap_id (No JUMPS): 15584\n",
      "2020-11-19 15:44:13.642718,PDP_philippines_20190707.csv,15640,15640,15584,219379,198452,196257\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190707.csv 2020-11-19 15:44:13.658748\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190702.csv 2020-11-19 15:44:13.658923\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190702.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190702.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190702.csv\n",
      "ap_id (all): 15923  ap_id (No JUMPS): 15868\n",
      "2020-11-19 16:02:57.849174,PDP_philippines_20190702.csv,15923,15923,15868,230387,207620,205307\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190702.csv 2020-11-19 16:02:57.865594\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190705.csv 2020-11-19 16:02:57.865744\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190705.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190705.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190705.csv\n",
      "ap_id (all): 16098  ap_id (No JUMPS): 16044\n",
      "2020-11-19 16:22:45.218667,PDP_philippines_20190705.csv,16098,16098,16044,240254,215325,212922\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190705.csv 2020-11-19 16:22:45.236548\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\t\t Start:  PDP_philippines_20190701.csv 2020-11-19 16:22:45.236680\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190701.csv  clean_by_aggregating_ts()\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/tmp/PDP_philippines_20190701.csv  filter_by_distance()\n",
      "log_file: /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/logs/1_PDP_philippines_20190701.csv\n",
      "ap_id (all): 15869  ap_id (No JUMPS): 15816\n",
      "2020-11-19 16:41:36.586460,PDP_philippines_20190701.csv,15869,15869,15816,227440,205160,202747\n",
      "Handled TS and Jump :  /mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190701.csv 2020-11-19 16:41:36.596243\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "csv preparation for map-matching input complete\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#raw_dir = '/mnt/lv/heromiya/PHLMobilityData/'    ## original raw file location\n",
    "#mapmatching_input_dir = '/mnt/lv/bidur/PHL_raw_data/clean_input/test/' ## prepare csv in required format(ap_id, ) for map-matching\n",
    "\n",
    "\n",
    "start_date = date(2019, 7, 1) # NPL_data_20190404.csv\n",
    "end_date = date(2019, 7, 7)\n",
    "    \n",
    "get_raw_files_main(raw_dir, mapmatching_input_dir, start_date , end_date)    \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "csv from __mapmatching_input_dir__ are processed and final csv are saved in __backup_dir__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190704.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190706.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190703.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190707.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190702.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190705.csv', '/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190701.csv']\n",
      "_____________________________________\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190704.csv START  2020-11-19 16:41:36.664344\n",
      "\n",
      "completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar import map-data/philippines-latest.osm.pbf\n",
      "<< multithreaded_process() -> START 2020-11-19 16:43:33.637658\n",
      "Very Few Data (<5 points) in <<  3480  >> ap_ids\n",
      "4400  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/1\n",
      "Very Few Data (<5 points) in <<  2908  >> ap_ids\n",
      "4972  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/0\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0\n",
      "\n",
      " Thread #1 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/1\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.res.gpx\n",
      "\n",
      " Thread #0 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/0\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.res.gpx\n",
      "Thread operation  complete\n",
      "\n",
      "\n",
      " Map Matching and route generation completed\n",
      "/mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/5_final_csv_4_mobmap.csv\n",
      "<< multithreaded_process() -> END 2020-11-19 17:52:23.284132\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190704.csv END  2020-11-19 17:52:23.426860\n",
      "Saved at:  /mnt/lv/bidur/PHL_raw_data/csv_4_mobmap/201907/PDP_philippines_20190704.csv\n",
      "__________________________________________________________________\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190706.csv START  2020-11-19 17:52:23.427125\n",
      "\n",
      "completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar import map-data/philippines-latest.osm.pbf\n",
      "<< multithreaded_process() -> START 2020-11-19 17:54:27.951369\n",
      "Very Few Data (<5 points) in <<  3592  >> ap_ids\n",
      "4466  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/1\n",
      "Very Few Data (<5 points) in <<  3056  >> ap_ids\n",
      "5001  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/0\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0\n",
      "\n",
      " Thread #1 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/1\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.res.gpx\n",
      "\n",
      " Thread #0 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/0\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.res.gpx\n",
      "Thread operation  complete\n",
      "\n",
      "\n",
      " Map Matching and route generation completed\n",
      "/mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/5_final_csv_4_mobmap.csv\n",
      "<< multithreaded_process() -> END 2020-11-19 19:04:07.849008\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190706.csv END  2020-11-19 19:04:08.075751\n",
      "Saved at:  /mnt/lv/bidur/PHL_raw_data/csv_4_mobmap/201907/PDP_philippines_20190706.csv\n",
      "__________________________________________________________________\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190703.csv START  2020-11-19 19:04:08.076246\n",
      "\n",
      "completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar import map-data/philippines-latest.osm.pbf\n",
      "<< multithreaded_process() -> START 2020-11-19 19:06:07.370835\n",
      "Very Few Data (<5 points) in <<  3494  >> ap_ids\n",
      "4524  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/0\n",
      "Very Few Data (<5 points) in <<  3219  >> ap_ids\n",
      "4799  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/1\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1\n",
      "\n",
      " Thread #1 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/1\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1/*.res.gpx\n",
      "\n",
      " Thread #0 completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar match /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.gpx\n",
      "\n",
      "convert_resgpx2csv -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/res_csv/0\n",
      "Matched gpx -> /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0/*.res.gpx\n",
      "Thread operation  complete\n",
      "\n",
      "\n",
      " Map Matching and route generation completed\n",
      "/mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/5_final_csv_4_mobmap.csv\n",
      "<< multithreaded_process() -> END 2020-11-19 20:16:38.706533\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190703.csv END  2020-11-19 20:16:38.935577\n",
      "Saved at:  /mnt/lv/bidur/PHL_raw_data/csv_4_mobmap/201907/PDP_philippines_20190703.csv\n",
      "__________________________________________________________________\n",
      "/mnt/lv/bidur/PHL_raw_data/clean_input/201907/PDP_philippines_20190707.csv START  2020-11-19 20:16:38.935847\n",
      "\n",
      "completed:  java -jar matching-web/target/graphhopper-map-matching-web-1.0-SNAPSHOT.jar import map-data/philippines-latest.osm.pbf\n",
      "<< multithreaded_process() -> START 2020-11-19 20:18:36.684925\n",
      "Very Few Data (<5 points) in <<  3621  >> ap_ids\n",
      "4156  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/1\n",
      "Very Few Data (<5 points) in <<  3094  >> ap_ids\n",
      "4682  csv file prepared and saved in  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/input/csv/0\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/1\n",
      "< apply_map_matching_multithread() >\n",
      "Current Working Directory  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master\n",
      "GPX_DIR:  /mnt/lv/bidur/PHL_raw_data/map_match_gps_data/map-matching-master/matching-web/src/test/resources/target/0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "arr_input_csv = get_all_files_from_dir(mapmatching_input_dir)\n",
    "\n",
    "print(arr_input_csv)\n",
    "print(\"_____________________________________\")\n",
    "for gps_csv in arr_input_csv:\n",
    "    \n",
    "    csv_path, csv_name = os.path.split(gps_csv)\n",
    "      \n",
    "    \n",
    "    if '20190703' in gps_csv:\n",
    "        continue\n",
    "    if '20190703' in gps_csv:\n",
    "        continue\n",
    "    if '20190703' in gps_csv:\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    print(csv_name, 'START ', str( datetime.now() ))\n",
    " \n",
    "    # 1. remove old data and create necessary directories\n",
    "    initialize()\n",
    "\n",
    "    # 2. ananymize ap_id column to int value ,   clip points within boundary\n",
    "    #gdf_probe_clipped, gdf_target = get_points_within_target_region (gps_csv, anonymize=True, display_plot = False)\n",
    "    gdf_probe_clipped, gdf_target = get_points_within_target_region (gps_csv, anonymize=False, display_plot = False)\n",
    "    \n",
    "    #print('----2 done----')\n",
    "    \n",
    "    \n",
    "    # 3. Preprocess: cleaning data & applying sampling\n",
    "    df_sample = preprocess_data()\n",
    "    #print('----3 done----')\n",
    "\n",
    "    # 4. map matching with osm roads using graphhopper\n",
    "    df_mapped_route = map_match_csv2gpx_multithread(df_sample) # multithreaded\n",
    "    \n",
    "    #Use this for normal execution without multi thread ( IF user permission do not allow multi-threading):\n",
    "    #df_mapped_route = map_match_csv2gpx(df_sample)\n",
    "    #print('----4 done----')\n",
    "    \n",
    "        \n",
    "    # 5. save final csv to backup dir\n",
    "    check_dir(backup_dir)# create directory if not exit\n",
    "    \n",
    "    final_data_path = backup_dir+csv_name\n",
    "    shutil.copyfile(map_matched_gps_probe,final_data_path)\n",
    "    #csv_path, csv_name = os.path.split(gps_csv)\n",
    "    #time_str = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "    #shutil.copyfile('/mnt/lv/bidur/PHL_raw_data/map_match_gps_data/output/time_dist_log.csv',backup_dir + '0log_'+ time_str+ csv_name)\n",
    "    \n",
    "    print(gps_csv, 'END ', str( datetime.now() ))\n",
    "    print('Saved at: ', final_data_path)\n",
    "    print('__________________________________________________________________')\n",
    "    \n",
    "\n",
    "print(\"ALL TASK COMPLETED!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%tb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Output summary for a sample inout for 20190703\n",
    "- about 9 hr 30 min \n",
    "- 10% = 525 ap_ids\n",
    "- 411 ap_ids in final output "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}