Merge branch 'master' into DE88_S3_ProdSink

db-updater/main.py

@@ -86,6 +86,7 @@
         sa.Column("predicted_off", TIMESTAMP_TZ()),
         sa.Column("predicted_on", TIMESTAMP_TZ()),
         sa.Column("predicted_in", TIMESTAMP_TZ()),
+        sa.Column("diverted", sa.Boolean),
     )
     VALID_EVENTS = {"arrival", "cancellation", "departure", "flightplan", "onblock", "offblock", "extendedFlightInfo", "flifo"}
 elif TABLE == "positions":
@@ -188,6 +189,7 @@
 cache_lock = threading.Lock()
 SQLITE_VAR_LIMIT = None
 
+dest_history = dict()
 
 class Cache(ABC):
     """A cache for accumulating flight or position information which can be flushed as necessary."""
@@ -368,6 +370,7 @@ def chunk(values: Iterable, chunk_size: Optional[int]) -> Iterator:
 
 def add_to_cache(data: dict) -> None:
     """add entry to the cache"""
+    check_for_new_destination(data)
     converted = convert_msg_fields(data)
     with cache_lock:
         cache.add(converted)
@@ -409,6 +412,8 @@ def process_unknown_message(data: dict) -> None:
 
 def process_arrival_message(data: dict) -> None:
     """Arrival message type"""
+    if "id" in data:
+        clear_dest_history(data["id"])
     return add_to_cache(data)
 
 
@@ -432,12 +437,16 @@ def process_offblock_message(data: dict) -> None:
 
 def process_onblock_message(data: dict) -> None:
     """Onblock message type"""
+    if "id" in data:
+        clear_dest_history(data["id"])
     data["actual_in"] = data["clock"]
     return add_to_cache(data)
 
 
 def process_flifo_message(data: dict) -> None:
     """flifo message type"""
+    # flightplan and flifo messages may contain a new dest indicating a diversion, flag it if so
+    check_for_diversions(data)
     # flifo messages try to help us with saner names, but we already convert
     # field names at the sqlalchemy level, so we actually need to convert the
     # nice names to ugly names so they can be converted again later...
@@ -465,6 +474,8 @@ def process_extended_flight_info_message(data: dict) -> None:
 
 def process_flightplan_message(data: dict) -> None:
     """Flightplan message type"""
+    # flightplan and flifo messages may contain a new dest indicating a diversion, flag it if so
+    check_for_diversions(data)
     disambiguate_altitude(data)
     return add_to_cache(data)
 
@@ -480,6 +491,41 @@ def process_keepalive_message(data: dict) -> None:
     print(f'Based on keepalive["pitr"], we are {behind} behind realtime')
 
 
+def clear_dest_history(flight) -> None:
+    """Remove id from dest_history"""
+    global dest_history
+    if flight in dest_history:
+        dest_history.pop(flight)
+    return
+
+
+def check_for_diversions(data: dict) -> None:
+    """ETMS message, check destination"""
+    global dest_history
+
+    dest = data.get("dest")
+    flight = data.get("id")
+    orig_dest = dest_history.get(flight)
+
+    if dest and flight:
+        if orig_dest and orig_dest != dest:
+            data["diverted"] = True
+
+        dest_history[flight] = dest
+
+    return
+
+def check_for_new_destination(data: dict) -> None:
+    """Look for a new destination that hasn't been seen by an ETMS message yet"""
+    global dest_history
+    dest = data.get("dest")
+    flight = data.get("id")
+
+    if dest and flight and flight not in dest_history:
+        dest_history[flight] = dest
+
+    return
+
 def disambiguate_altitude(data: dict):
     """Replaces the alt field in the passed dict with an unambiguous field name"""
 

db-updater/test/test_db_updater.py

@@ -30,6 +30,7 @@ class TestInsertAndExpire(unittest.TestCase):
         b'{"pitr": "1589549554", "type": "extendedFlightInfo", "ident": "SKW3284", "actual_departure_gate": "G11", "actual_departure_terminal": "3", "actual_out": "1589547780", "estimated_arrival_gate": "C3", "estimated_in": "1589553900", "estimated_out": "1589548200", "facility_hash": "F44B2C6C456D33FB", "facility_name": "Airline", "id": "SKW3284-1589345159-airline-0376", "scheduled_departure_terminal": "3", "scheduled_in": "1589554920", "scheduled_out": "1589548200"}',
         b'{"pitr": "1589551845", "type": "offblock", "ident": "UAL2465", "clock": "1589551845", "dest": "KDEN", "facility_hash": "23D67E4254EC60CD", "facility_name": "United Airlines", "id": "UAL2465-1589328337-fa-0001", "orig": "KORD"}',
         b'{"pitr": "1589551558", "type": "onblock", "ident": "CHH7691", "clock": "1589551547", "dest": "ZSOF", "facility_hash": "F44B2C6C456D33FB", "facility_name": "Airline", "id": "CHH7691-1589345160-airline-0022", "orig": "ZJSY"}',
+        b'{"pitr": "1589551880", "type": "flifo", "ident": "UAL2465", "clock": "1589551845", "dest": "KHOU", "facility_hash": "23D67E4254EC60CD", "facility_name": "United Airlines", "id": "UAL2465-1589328337-fa-0001", "orig": "KORD"}',
     ]
     flight_msgs = [FakeMessage(msg) for msg in _flight_msgs]
 
@@ -44,7 +45,7 @@ class TestInsertAndExpire(unittest.TestCase):
     ]
     position_msgs = [FakeMessage(msg) for msg in _position_msgs]
 
-    expected_flights_results = [('CGEYQ-1589545563-3-1-67', 'CGEYQ', 'CGEYQ', None, None, 'CYZX', 'CYQI', 'C177', None, 112, None, 6000, None, 'CYZX..YZX..MUXEL..OMTIV..CYQI', 'F', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, datetime.datetime(2020, 5, 15, 13, 15), None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None), ('BLOCKED-1589549420-adhoc-0', 'BLOCKED', 'BLOCKED', None, None, 'KIWA', None, 'C172', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30, 20), None, None, None, None, None, None, None, None, None, None, None, None, None), ('BMJ64-1589543130-5-1-174', 'BMJ64', 'N104BA', None, None, 'KLJF', 'KBRD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 27, 1), None, None, None, None, None, None, None, None, None, None, None, None), ('CES2187-1589431537-airline-0180', 'CES2187', None, None, None, 'ZLXY', 'ZSAM', 'A320', None, 299, None, None, True, None, 'X', None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 1, 26), None, None, None, None, None, datetime.datetime(2020, 5, 16, 5, 5), datetime.datetime(2020, 5, 16, 7, 40), None, datetime.datetime(2020, 5, 16, 5, 5), None, None, None, None, None, None), ('SKW3284-1589345159-airline-0376', 'SKW3284', None, None, None, None, None, None, None, None, None, None, None, None, None, None, 'C3', 'G11', None, None, None, '3', '3', None, None, None, None, datetime.datetime(2020, 5, 15, 13, 3), None, None, None, datetime.datetime(2020, 5, 15, 13, 10), None, None, datetime.datetime(2020, 5, 15, 14, 45), None, datetime.datetime(2020, 5, 15, 13, 10), datetime.datetime(2020, 5, 15, 15, 2), None, None, None, None), ('UAL2465-1589328337-fa-0001', 'UAL2465', None, None, None, 'KORD', 'KDEN', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 10, 45), None, None, None, None, None, None, None, None, None, None, None, None, None, None), ('CHH7691-1589345160-airline-0022', 'CHH7691', None, None, None, 'ZJSY', 'ZSOF', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 5, 47), None, None, None, None, None, None, None, None, None, None, None)]
+    expected_flights_results = [('CGEYQ-1589545563-3-1-67', 'CGEYQ', 'CGEYQ', None, None, 'CYZX', 'CYQI', 'C177', None, 112, None, 6000, None, 'CYZX..YZX..MUXEL..OMTIV..CYQI', 'F', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, datetime.datetime(2020, 5, 15, 13, 15), None, None, None, datetime.datetime(2020, 5, 15, 13, 30), datetime.datetime(2020, 5, 15, 14, 18, 58), None, None), ('BLOCKED-1589549420-adhoc-0', 'BLOCKED', 'BLOCKED', None, None, 'KIWA', None, 'C172', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 30, 20), None, None, None, None, None, None, None, None, None, None, None, None, None, None), ('BMJ64-1589543130-5-1-174', 'BMJ64', 'N104BA', None, None, 'KLJF', 'KBRD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 13, 27, 1), None, None, None, None, None, None, None, None, None, None, None, None, None), ('CES2187-1589431537-airline-0180', 'CES2187', None, None, None, 'ZLXY', 'ZSAM', 'A320', None, 299, None, None, True, None, 'X', None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 1, 26), None, None, None, None, None, datetime.datetime(2020, 5, 16, 5, 5), datetime.datetime(2020, 5, 16, 7, 40), None, datetime.datetime(2020, 5, 16, 5, 5), None, None, None, None, None, None, None), ('SKW3284-1589345159-airline-0376', 'SKW3284', None, None, None, None, None, None, None, None, None, None, None, None, None, None, 'C3', 'G11', None, None, None, '3', '3', None, None, None, None, datetime.datetime(2020, 5, 15, 13, 3), None, None, None, datetime.datetime(2020, 5, 15, 13, 10), None, None, datetime.datetime(2020, 5, 15, 14, 45), None, datetime.datetime(2020, 5, 15, 13, 10), datetime.datetime(2020, 5, 15, 15, 2), None, None, None, None, None), ('UAL2465-1589328337-fa-0001', 'UAL2465', None, None, None, 'KORD', 'KHOU', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 10, 45), None, None, None, None, None, None, None, None, None, None, None, None, None, None, 1), ('CHH7691-1589345160-airline-0022', 'CHH7691', None, None, None, 'ZJSY', 'ZSOF', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, datetime.datetime(2020, 5, 15, 14, 5, 47), None, None, None, None, None, None, None, None, None, None, None, None)]
     expected_positions_results = [('UEA2236-1591584600-schedule-0391', datetime.datetime(2020, 6, 10, 4, 36, 7), '29.84762', '104.34529', 'E4030EF8218DE21D', 'FlightAware ADS-B', 'A', None, None, 13800, None, ' ', None, 'B9985', 'ZGGG', 'ZUUU', None, None, None, None, 319, '336', '336.1', None, '780BBB', '0.484', None, None, None, None, None, None, None, None, None, None, None, None, 600, None, 250, 314, 4525, 4, 1, None, None, None, None, None, None), ('CES5489-1591589400-schedule-0472', datetime.datetime(2020, 6, 10, 4, 36, 3), '31.43910', '115.68351', '18249C69E5BC3F1B', 'FlightAware ADS-B', 'A', None, None, 17700, None, ' ', None, 'B1610', 'ZSOF', 'ZPPP', None, None, None, None, 401, '244', '250.0', None, '780D1B', '0.648', None, None, None, None, None, None, None, None, None, None, None, None, 512, None, 312, 414, 5005, -4, 1, None, None, None, None, None, None), ('CAL782-1591588500-schedule-0187', datetime.datetime(2020, 6, 10, 4, 36, 20), '12.81155', '108.19659', '52446C6CE8E0B859', 'FlightAware ADS-B', 'A', None, None, 34500, None, 'C', None, 'B18909', 'VVTS', 'RCTP', None, None, None, None, 494, '36', '37.3', None, '899128', '0.852', None, None, None, None, None, None, None, None, None, None, None, None, 244, None, 297, 510, 5620, -37, 1, None, None, None, None, None, None), ('CWA926-1591760300-2-0-98', datetime.datetime(2020, 6, 10, 4, 36, 30), '57.87987', '-116.59763', 'B566E7DB03CCFF2C', 'FlightAware ADS-B', 'A', None, None, 23575, None, 'C', None, 'CGLUQ', 'CYOJ', 'CYEG', None, None, None, None, 234, '158', None, None, 'C063CD', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 5213, None, None, None, None, None, None, None, None), ('SOO7930-1591724581-7-3-192', datetime.datetime(2020, 6, 10, 4, 36, 49), '64.62843', '-143.16364', 'C3DCCEF85E56D79D', 'FlightAware ADS-B', 'A', None, 'B77L', 34000, None, ' ', None, 'N702GT', 'KJFK', 'RKSI', None, None, None, None, 494, '272', None, None, 'A95CA4', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 3064, None, None, None, None, None, None, None, None), ('CES5489-1591589400-schedule-0472', datetime.datetime(2020, 6, 10, 4, 36, 59), '29.50584', '106.56178', 'E4030EF8218DE21D', 'FlightAware ADS-B', 'A', None, None, 38100, None, ' ', None, 'B1610', 'ZSOF', 'ZUUU', None, None, None, None, 464, '179', '184.9', None, '780D1B', '0.804', None, None, None, None, None, None, None, None, None, None, None, None, 205, None, 255, 470, 4113, -48, 1, None, None, None, None, None, None), ('N517MT-1591762834-adhoc-0', datetime.datetime(2020, 6, 10, 4, 37, 8), '33.58891', '-97.21146', 'A4C87376ACECA2CE', 'FlightAware ADS-B', 'A', None, 'EC35', 1550, None, 'D', None, 'N517MT', 'KDTO', None, None, None, None, None, 119, '13', None, None, 'A67C95', None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, 1200, None, None, None, None, None, None, None, None)]
     # fmt: on
 

fids/requirements/base.txt

@@ -4,9 +4,9 @@
 #
 #    pip-compile --generate-hashes --output-file=base.txt base.in
 #
-certifi==2021.10.8 \
-    --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
-    --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
+certifi==2022.12.7 \
+    --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
+    --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
     # via requests
 charset-normalizer==2.0.10 \
     --hash=sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd \

map/requirements/base.in

@@ -1,2 +1,2 @@
-Flask==1.1.4
+Flask==2.3.2
 confluent-kafka==1.8.2