Merge pull request #185 from gm3dmo/timings

Adding script timings

scripts/insert-all-cemeteries.py

@@ -4,22 +4,23 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Cemetery
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/cemetery.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
     reader.fieldnames = [field.replace('.', '_') for field in reader.fieldnames]
 
     # add a country model for each row in the csv file
+    start_insert_time = time.time()
     for row in reader:
         if row['latitude'] == '':
             row['latitude'] = 0 
@@ -36,3 +37,8 @@ def run():
         except Exception as e:
             print(f"""💥row: ({row}) """)
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-companies.py

@@ -4,21 +4,21 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Company
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/company.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
 
-    print(reader.fieldnames)
+    start_insert_time = time.time()
     for row in reader:
         #print(row['name'])
         try:
@@ -29,3 +29,7 @@ def run():
             print(f"""💥row: ({row}) """)
             raise e
 
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-countries.py

@@ -4,25 +4,22 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Country
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
-    #ref_data_url = "https://raw.githubusercontent.com/mledoze/countries/master/dist/countries.csv"
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/country.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
-    #reader.fieldnames = [field.replace('.', '_') for field in reader.fieldnames]
 
-    print(reader.fieldnames)
+    start_insert_time = time.time()
     for row in reader:
-        print(row['Name'])
         try:
             Country.objects.create(
                 id = row['id'],
@@ -36,3 +33,7 @@ def run():
             print("Error with: " + row['Name'])
             raise e
 
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-decorations.py

@@ -4,21 +4,22 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Decoration
 
+
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/decoration.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
 
-    # add a country model for each row in the csv file
+    start_insert_time = time.time()
     for row in reader:
         #print(row['id'])
         try:
@@ -31,3 +32,8 @@ def run():
         except Exception as e:
             print("Error with: " + row['name'])
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-pow-camps.py

@@ -4,23 +4,23 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import PowCamp
 
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/pow-camp.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
 
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
 
-    # add a country model for each row in the csv file
+    start_insert_time = time.time()
     for row in reader:
         #print(f""" {row['id']} {row['Name']} ({row['PresentCountry_id']}) {row['WartimeCountry']} {row['Latitude']} {row['Longitude']}""")
         try:
@@ -36,3 +36,8 @@ def run():
             print(f"""💥row: ({row}) """)
             raise e
 
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-ranks.py

@@ -4,21 +4,21 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Rank
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/rank.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
 
-    # add a country model for each row in the csv file
+    start_insert_time = time.time()
     for row in reader:
         #print(row['name'])
         try:
@@ -31,3 +31,8 @@ def run():
         except Exception as e:
             print(f"""💥row: ({row}) """)
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-soldier-deaths.py

@@ -4,22 +4,24 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import SoldierDeath
     from cmp.models import Company
 
+
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-death.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
+
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
 
-    # add a country model for each row in the csv file
+    start_insert_time = time.time()
     for row in reader:
         #print(f"""row: ({row['id']}) cwgc:({row['cwgc_id']})""")
         try:
@@ -41,3 +43,8 @@ def run():
         except Exception as e:
             print(f"""💥row: ({row['id']}) cwgc:({row['cwgc_id']}) company:({row['company_id']}) cemetery:({row['cemetery_id']})""")
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-soldier-decorations.py

@@ -4,23 +4,24 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import SoldierDecoration
     from cmp.models import Company
     from cmp.models import Country
     from cmp.models import Soldier
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-decoration.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('ISO-8859-1').splitlines())
 
+    start_insert_time = time.time()
     for row in reader:
         #print(f"""row: ({row['id']}) cwgc:({row['cwgc_id']})""")
         try:
@@ -61,3 +62,8 @@ def run():
         except Exception as e:
             print(f"""💥row: {row}""")
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-soldier-imprisonments.py

@@ -4,21 +4,21 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import SoldierImprisonment
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier-imprisonment.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('utf-8').splitlines())
     # breakpoint()
-    # id,soldier_id,company_id,powNumber,powCamp_id,dateFrom,dateTo,notes
+    start_insert_time = time.time()
     for row in reader:
         #print(f"""SoldierImprisonment: {row['id']} {row['soldier_id']} """)
         try:
@@ -35,3 +35,9 @@ def run():
         except Exception as e:
             print(f"""💥row: ({row}) """)
             raise e
+
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")

scripts/insert-all-soldiers.py

@@ -4,22 +4,21 @@ def run():
     import sys
     import urllib3
     import csv
+    import time
     from cmp.models import Soldier
 
     print()
     title = sys.argv[2]
-    print(f"""\033[4;33m{title}\033[0m""")
-    print("-" * len(title))
 
+    start_fetch_time = time.time()
     ref_data_url = "https://raw.githubusercontent.com/gm3dmo/old-cmp/main/data/soldier.csv"
     http = urllib3.PoolManager()
     r = http.request('GET', ref_data_url)
-    print(f"""Fetch table response code: {r.status}""")
+    end_fetch_time = time.time()
     # load the response into a csv dictionary reader
     reader = csv.DictReader(r.data.decode('ISO-8859-1').splitlines())
     # breakpoint()
-    print(reader) 
-    # print(reader.fieldnames)
+    start_insert_time = time.time()
     for row in reader:
         # id,surname,initials,army_number,rank_id,notes
         #print(f"""{row['id']} {row['surname']}""")
@@ -36,3 +35,8 @@ def run():
             print(f"""💥row: ({row}) """)
 
             raise e
+
+    end_insert_time = time.time()
+    time_to_fetch = end_fetch_time - start_fetch_time
+    time_to_insert = end_insert_time - start_insert_time
+    print(f"""\033[4;33m{title}\033[0m Fetch table response code: {r.status} time (seconds) to fetch: {time_to_fetch:.2f} time to insert {time_to_insert:.2f}""")