From 14f1025f25eb2ffa01739225e793a798f78ffac5 Mon Sep 17 00:00:00 2001 From: David Morris Date: Sun, 2 Jun 2024 18:02:18 +0100 Subject: [PATCH] Adding script timings --- scripts/insert-all-cemeteries.py | 12 +++++++++--- scripts/insert-all-companies.py | 12 ++++++++---- scripts/insert-all-countries.py | 15 ++++++++------- scripts/insert-all-decorations.py | 14 ++++++++++---- scripts/insert-all-pow-camps.py | 13 +++++++++---- scripts/insert-all-ranks.py | 13 +++++++++---- scripts/insert-all-soldier-deaths.py | 15 +++++++++++---- scripts/insert-all-soldier-decorations.py | 12 +++++++++--- scripts/insert-all-soldier-imprisonments.py | 14 ++++++++++---- scripts/insert-all-soldiers.py | 14 +++++++++----- 10 files changed, 92 insertions(+), 42 deletions(-) diff --git a/scripts/insert-all-cemeteries.py b/scripts/insert-all-cemeteries.py index d08e415..647fb25 100644 --- a/scripts/insert-all-cemeteries.py +++ b/scripts/insert-all-cemeteries.py @@ -4,22 +4,23 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Cemetery print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/cemetery.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) reader.fieldnames = [field.replace('.', '_') for field in reader.fieldnames] # add a country model for each row in the csv file + start_insert_time = time.time() for row in reader: if row['latitude'] == '': row['latitude'] = 0 @@ -36,3 +37,8 @@ def run(): except Exception as e: print(f"""💥row: ({row}) """) raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-companies.py b/scripts/insert-all-companies.py index 0efefa5..4847f38 100644 --- a/scripts/insert-all-companies.py +++ b/scripts/insert-all-companies.py @@ -4,21 +4,21 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Company print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/company.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - print(reader.fieldnames) + start_insert_time = time.time() for row in reader: #print(row['name']) try: @@ -29,3 +29,7 @@ def run(): print(f"""💥row: ({row}) """) raise e + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") diff --git a/scripts/insert-all-countries.py b/scripts/insert-all-countries.py index 8e2bae8..64a5a8d 100644 --- a/scripts/insert-all-countries.py +++ b/scripts/insert-all-countries.py @@ -4,25 +4,22 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Country print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) - #ref_data_url = "https://raw.githubusercontent.com/mledoze/countries/master/dist/countries.csv" + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/country.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - #reader.fieldnames = [field.replace('.', '_') for field in reader.fieldnames] - print(reader.fieldnames) + start_insert_time = time.time() for row in reader: - print(row['Name']) try: Country.objects.create( id = row['id'], @@ -36,3 +33,7 @@ def run(): print("Error with: " + row['Name']) raise e + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") diff --git a/scripts/insert-all-decorations.py b/scripts/insert-all-decorations.py index ab23546..0976a03 100644 --- a/scripts/insert-all-decorations.py +++ b/scripts/insert-all-decorations.py @@ -4,21 +4,22 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Decoration + print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/decoration.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - # add a country model for each row in the csv file + start_insert_time = time.time() for row in reader: #print(row['id']) try: @@ -31,3 +32,8 @@ def run(): except Exception as e: print("Error with: " + row['name']) raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-pow-camps.py b/scripts/insert-all-pow-camps.py index e098afa..57741a7 100644 --- a/scripts/insert-all-pow-camps.py +++ b/scripts/insert-all-pow-camps.py @@ -4,23 +4,23 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import PowCamp print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/pow-camp.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - # add a country model for each row in the csv file + start_insert_time = time.time() for row in reader: #print(f""" {row['id']} {row['Name']} ({row['PresentCountry_id']}) {row['WartimeCountry']} {row['Latitude']} {row['Longitude']}""") try: @@ -36,3 +36,8 @@ def run(): print(f"""💥row: ({row}) """) raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-ranks.py b/scripts/insert-all-ranks.py index 0057671..f39c26d 100644 --- a/scripts/insert-all-ranks.py +++ b/scripts/insert-all-ranks.py @@ -4,21 +4,21 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Rank print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/rank.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - # add a country model for each row in the csv file + start_insert_time = time.time() for row in reader: #print(row['name']) try: @@ -31,3 +31,8 @@ def run(): except Exception as e: print(f"""💥row: ({row}) """) raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-soldier-deaths.py b/scripts/insert-all-soldier-deaths.py index 908848d..a52e08f 100644 --- a/scripts/insert-all-soldier-deaths.py +++ b/scripts/insert-all-soldier-deaths.py @@ -4,22 +4,24 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import SoldierDeath from cmp.models import Company + print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-death.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() + # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) - # add a country model for each row in the csv file + start_insert_time = time.time() for row in reader: #print(f"""row: ({row['id']}) cwgc:({row['cwgc_id']})""") try: @@ -41,3 +43,8 @@ def run(): except Exception as e: print(f"""💥row: ({row['id']}) cwgc:({row['cwgc_id']}) company:({row['company_id']}) cemetery:({row['cemetery_id']})""") raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-soldier-decorations.py b/scripts/insert-all-soldier-decorations.py index 4b00c3d..439d331 100644 --- a/scripts/insert-all-soldier-decorations.py +++ b/scripts/insert-all-soldier-decorations.py @@ -4,6 +4,7 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import SoldierDecoration from cmp.models import Company from cmp.models import Country @@ -11,16 +12,16 @@ def run(): print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-decoration.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('ISO-8859-1').splitlines()) + start_insert_time = time.time() for row in reader: #print(f"""row: ({row['id']}) cwgc:({row['cwgc_id']})""") try: @@ -61,3 +62,8 @@ def run(): except Exception as e: print(f"""💥row: {row}""") raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-soldier-imprisonments.py b/scripts/insert-all-soldier-imprisonments.py index 69949d6..580aacc 100644 --- a/scripts/insert-all-soldier-imprisonments.py +++ b/scripts/insert-all-soldier-imprisonments.py @@ -4,21 +4,21 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import SoldierImprisonment print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-imprisonment.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('utf-8').splitlines()) # breakpoint() - # id,soldier_id,company_id,powNumber,powCamp_id,dateFrom,dateTo,notes + start_insert_time = time.time() for row in reader: #print(f"""SoldierImprisonment: {row['id']} {row['soldier_id']} """) try: @@ -35,3 +35,9 @@ def run(): except Exception as e: print(f"""💥row: ({row}) """) raise e + + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file diff --git a/scripts/insert-all-soldiers.py b/scripts/insert-all-soldiers.py index 3695578..94c8489 100644 --- a/scripts/insert-all-soldiers.py +++ b/scripts/insert-all-soldiers.py @@ -4,22 +4,21 @@ def run(): import sys import urllib3 import csv + import time from cmp.models import Soldier print() title = sys.argv[2] - print(f"""\033[4;33m{title}\033[0m""") - print("-" * len(title)) + start_fetch_time = time.time() ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier.csv" http = urllib3.PoolManager() r = http.request('GET', ref_data_url) - print(f"""Fetch table response code: {r.status}""") + end_fetch_time = time.time() # load the response into a csv dictionary reader reader = csv.DictReader(r.data.decode('ISO-8859-1').splitlines()) # breakpoint() - print(reader) - # print(reader.fieldnames) + start_insert_time = time.time() for row in reader: # id,surname,initials,army_number,rank_id,notes #print(f"""{row['id']} {row['surname']}""") @@ -36,3 +35,8 @@ def run(): print(f"""💥row: ({row}) """) raise e + + end_insert_time = time.time() + time_to_fetch = end_fetch_time - start_fetch_time + time_to_insert = end_insert_time - start_insert_time + print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""") \ No newline at end of file