From 7839951766cd285fb9d9164ed0883fc90a9be216 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 10:41:17 -0700
Subject: [PATCH 01/21] Init commit

Adding TestOverpass.py
---
 emission/individual_tests/TestOverpass.py | 58 +++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 emission/individual_tests/TestOverpass.py

diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
new file mode 100644
index 000000000..f028d6817
--- /dev/null
+++ b/emission/individual_tests/TestOverpass.py
@@ -0,0 +1,58 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import *
+import unittest
+import os
+import requests
+import emission.net.ext_service.transit_matching.match_stops as enetm
+
+#Set up query
+OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
+
+#Sample loc1 = NREL East Gate
+loc1 = {'coordinates': [-105.16844103184974, 39.740428870224605]}
+#Sample loc2 = Denver Union Station
+loc2 = {'coordinates': [-105.00083982302972, 39.753710532185025]}
+#Sample loc3 = Grand Junction Train Station, CO
+loc3 = {'coordinates': [-108.57055213129632, 39.06472424640481]}
+
+class OverpassTest(unittest.TestCase):
+    def setUp(self):
+        sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
+        call_base = 'api/interpreter?data='
+        self.de_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
+        self.gfbk_url_base = 'http://overpass.geofabrik.de/' + OVERPASS_KEY + '/' + call_base + sample_data
+
+    def test_overpass(self):
+        r_gfbk = requests.get(self.gfbk_url_base)
+        r_de = requests.get(self.de_url_base)
+        
+        if r_gfbk.status_code == 200 and r_de.status_code == 200:
+            print("requests successful!")
+            r_gfbk_len, r_de_len = len(r_gfbk.json()), len(r_de.json())
+            self.assertEqual(r_gfbk_len, r_de_len)
+        else:
+            print("status_gfbk", r_gfbk.status_code, type(r_gfbk.status_code), "status_de", r_de.status_code)
+
+    #Test utilizes the functions get_stops_near, get_public_transit_stops, and make_request_and_catch.  
+    def test_get_stops_near(self):
+        actual_result = enetm.get_stops_near(loc1, 150.0)[0]['routes'][0]['tags']
+        expected_result = {'from': 'National Renewable Energy Lab', 'name': 'RTD Route 125: Red Rocks College', 'network': 'RTD', 'network:wikidata': 'Q7309183', 'network:wikipedia': 'en:Regional Transportation District', 'operator': 'Regional Transportation District', 'public_transport:version': '1', 'ref': '125', 'route': 'bus', 'to': 'Red Rocks College', 'type': 'route'}
+        self.assertEqual(expected_result, actual_result)
+   
+    #Get_stops_near generates two stops from the given coordinates.
+    # Get_predicted_transit_mode finds a common route between them (train).
+    def test_get_predicted_transit_mode(self):
+        stop1 = enetm.get_stops_near(loc2, 400.0)
+        stop2 = enetm.get_stops_near(loc3, 400.0)
+        actual_result = enetm.get_predicted_transit_mode(stop1, stop2)
+        expected_result = ['train', 'train']
+        self.assertEqual(actual_result, expected_result)
+
+if __name__ == '__main__':
+    unittest.main()
+

From b1b0698a4e12890366b7f06599e7d4bf411e0adc Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 11:56:16 -0700
Subject: [PATCH 02/21] Adding shell script and workflow file

Workflow file runs the shell script.

Shell script sets up emission environment and runs TestOverpass.py.
---
 .github/workflows/test-overpass.yml           | 29 +++++++++++++++++++
 .../individual_tests/setup_overpass_tests.sh  | 29 +++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 .github/workflows/test-overpass.yml
 create mode 100644 emission/individual_tests/setup_overpass_tests.sh

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
new file mode 100644
index 000000000..270ff5c2c
--- /dev/null
+++ b/.github/workflows/test-overpass.yml
@@ -0,0 +1,29 @@
+name: overpass-test
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches:
+      - overpass
+  # schedule:
+
+  #   # Run every Sunday at 6:05 am
+  #   - cron:  '5 6 * * 0'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    # Tests runner is functional
+    - name: Workflow test
+      run: echo Good morning!
+      
+      # Runs TestOverpass.py to check for API functionality
+    - name: Test Overpass
+      env:
+        OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
+      run: ../../emission/individual_tests/setup_overpass_tests.sh
+      
diff --git a/emission/individual_tests/setup_overpass_tests.sh b/emission/individual_tests/setup_overpass_tests.sh
new file mode 100644
index 000000000..b224bd4f8
--- /dev/null
+++ b/emission/individual_tests/setup_overpass_tests.sh
@@ -0,0 +1,29 @@
+#Set up the testing environment
+# Using an automated install 
+cd /src/e-mission-server
+
+#set database URL using environment variable
+echo ${DB_HOST}
+if [ -z ${DB_HOST} ] ; then
+    local_host=`hostname -i`
+    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
+else
+    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
+fi
+cat conf/storage/db.conf
+
+echo "Setting up conda..."
+source setup/setup_conda.sh Linux-x86_64
+
+echo "Setting up the test environment..."
+source setup/setup_tests.sh
+
+echo "Running tests..."
+source setup/activate_tests.sh
+
+echo "Adding permissions for the runIntegrationTests.sh script"
+chmod +x runIntegrationTests.sh
+echo "Permissions added for the runIntegrationTests.sh script"
+
+set -e
+PYTHONPATH=. python -m unittest emission/individual_tests/TestOverpass.py

From 6857bb1a1be59557582a60998c63d458fe3b5977 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 12:05:27 -0700
Subject: [PATCH 03/21] Updating workflow

Adding commands to workflow to get a sense of why it's not running.
---
 .github/workflows/test-overpass.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index 270ff5c2c..f42c24aaa 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -25,5 +25,9 @@ jobs:
     - name: Test Overpass
       env:
         OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
-      run: ../../emission/individual_tests/setup_overpass_tests.sh
+      run: |
+        echo $OVERPASS_KEY
+        pwd
+        ls
+        ../emission/individual_tests/setup_overpass_tests.sh
       

From 38a666a4e2a446028a1842a7f3f58c22169bc847 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 12:15:17 -0700
Subject: [PATCH 04/21] Getting the directory right

Running with the correct path this time (I think)
---
 .github/workflows/test-overpass.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index f42c24aaa..ae0f0b0d9 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -26,8 +26,6 @@ jobs:
       env:
         OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
       run: |
-        echo $OVERPASS_KEY
-        pwd
-        ls
-        ../emission/individual_tests/setup_overpass_tests.sh
+        echo Testing!
+        ./emission/individual_tests/setup_overpass_tests.sh
       

From 1f7c3f1af6ad05ce3134aff0a90d7821a11ddf51 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 12:23:45 -0700
Subject: [PATCH 05/21] Adding permissions?

Adding runner permissions for setup script.
---
 .github/workflows/test-overpass.yml               | 2 +-
 emission/individual_tests/setup_overpass_tests.sh | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index ae0f0b0d9..3ff2ab3e5 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -27,5 +27,5 @@ jobs:
         OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
       run: |
         echo Testing!
+        chmod +x setup_overpass_tests.sh
         ./emission/individual_tests/setup_overpass_tests.sh
-      
diff --git a/emission/individual_tests/setup_overpass_tests.sh b/emission/individual_tests/setup_overpass_tests.sh
index b224bd4f8..98aeebc9d 100644
--- a/emission/individual_tests/setup_overpass_tests.sh
+++ b/emission/individual_tests/setup_overpass_tests.sh
@@ -21,9 +21,5 @@ source setup/setup_tests.sh
 echo "Running tests..."
 source setup/activate_tests.sh
 
-echo "Adding permissions for the runIntegrationTests.sh script"
-chmod +x runIntegrationTests.sh
-echo "Permissions added for the runIntegrationTests.sh script"
-
 set -e
 PYTHONPATH=. python -m unittest emission/individual_tests/TestOverpass.py

From 6fde585825c8974a5df6839c000ac6f83ced5a2c Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Wed, 7 Feb 2024 12:25:23 -0700
Subject: [PATCH 06/21] Silly

Didn't set correct path to file to add permissions.
---
 .github/workflows/test-overpass.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index 3ff2ab3e5..fd64d8fed 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -27,5 +27,5 @@ jobs:
         OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
       run: |
         echo Testing!
-        chmod +x setup_overpass_tests.sh
+        chmod +x emission/individual_tests/setup_overpass_tests.sh
         ./emission/individual_tests/setup_overpass_tests.sh

From 177f78dddbe6237fbdc8093f626c03410d572000 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 16:34:15 -0600
Subject: [PATCH 07/21] Delete overpass_server.json.sample

---
 conf/net/ext_service/overpass_server.json.sample | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100644 conf/net/ext_service/overpass_server.json.sample

diff --git a/conf/net/ext_service/overpass_server.json.sample b/conf/net/ext_service/overpass_server.json.sample
deleted file mode 100644
index 22fd9c49e..000000000
--- a/conf/net/ext_service/overpass_server.json.sample
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "__comment": "server running the overpass API to query OSM (e.g. https://wiki.openstreetmap.org/wiki/Overpass_API) without the /api/interpreter",
-    "url": "https://lz4.overpass-api.de/"
-}

From 41a9785967f6a0320a2ca5bdb68d3abbd7885589 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 16:37:16 -0600
Subject: [PATCH 08/21] Integrate overpass API

Change that integrates the paid overpass API with not only the integration test, but the overall functionality of match_stops.py
---
 .../net/ext_service/transit_matching/match_stops.py   | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index d08505e9e..41d9dc80c 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -3,14 +3,16 @@
 import requests
 import attrdict as ad
 import itertools
-import copy
+import os
 import time
 
 try:
-    config_file = open('conf/net/ext_service/overpass_server.json')
+    OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
+    url = 'http://overpass.geofabrik.de/' + OVERPASS_KEY + '/'
+    print("overpass configured")
 except:
     print("overpass not configured, falling back to default overleaf.de")
-    config_file = open('conf/net/ext_service/overpass_server.json.sample')
+    url = "https://lz4.overpass-api.de/"
 
 try:
     query_file = open('conf/net/ext_service/overpass_transit_stops_query_template')
@@ -18,9 +20,6 @@
     print("transit stops query not configured, falling back to default")
     query_file = open('conf/net/ext_service/overpass_transit_stops_query_template.sample')
 
-config_data = json.load(config_file)
-url = config_data["url"]
-
 query_string = "".join(query_file.readlines())
 
 RETRY = -1

From aee54d1492fc6e60faa54e15355ebc2a37e1f740 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 17:11:19 -0600
Subject: [PATCH 09/21] Remove db host check + prints

Figuring out why tests are passing locally and failing on github actions with prints.
---
 emission/individual_tests/TestOverpass.py                | 2 ++
 emission/individual_tests/setup_overpass_tests.sh        | 6 ------
 emission/net/ext_service/transit_matching/match_stops.py | 1 +
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
index f028d6817..7cffb197e 100644
--- a/emission/individual_tests/TestOverpass.py
+++ b/emission/individual_tests/TestOverpass.py
@@ -41,6 +41,7 @@ def test_overpass(self):
     #Test utilizes the functions get_stops_near, get_public_transit_stops, and make_request_and_catch.  
     def test_get_stops_near(self):
         actual_result = enetm.get_stops_near(loc1, 150.0)[0]['routes'][0]['tags']
+        print("ACTUAL_RESULT:", actual_result, type(actual_result))
         expected_result = {'from': 'National Renewable Energy Lab', 'name': 'RTD Route 125: Red Rocks College', 'network': 'RTD', 'network:wikidata': 'Q7309183', 'network:wikipedia': 'en:Regional Transportation District', 'operator': 'Regional Transportation District', 'public_transport:version': '1', 'ref': '125', 'route': 'bus', 'to': 'Red Rocks College', 'type': 'route'}
         self.assertEqual(expected_result, actual_result)
    
@@ -50,6 +51,7 @@ def test_get_predicted_transit_mode(self):
         stop1 = enetm.get_stops_near(loc2, 400.0)
         stop2 = enetm.get_stops_near(loc3, 400.0)
         actual_result = enetm.get_predicted_transit_mode(stop1, stop2)
+        print("ACTUAL TRANSIT MODE: ", actual_result)
         expected_result = ['train', 'train']
         self.assertEqual(actual_result, expected_result)
 
diff --git a/emission/individual_tests/setup_overpass_tests.sh b/emission/individual_tests/setup_overpass_tests.sh
index 98aeebc9d..0d5b21474 100644
--- a/emission/individual_tests/setup_overpass_tests.sh
+++ b/emission/individual_tests/setup_overpass_tests.sh
@@ -4,12 +4,6 @@ cd /src/e-mission-server
 
 #set database URL using environment variable
 echo ${DB_HOST}
-if [ -z ${DB_HOST} ] ; then
-    local_host=`hostname -i`
-    sed "s_localhost_${local_host}_" conf/storage/db.conf.sample > conf/storage/db.conf
-else
-    sed "s_localhost_${DB_HOST}_" conf/storage/db.conf.sample > conf/storage/db.conf
-fi
 cat conf/storage/db.conf
 
 echo "Setting up conda..."
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 41d9dc80c..004dfd486 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -26,6 +26,7 @@
 
 def make_request_and_catch(overpass_query):
     try:
+        print("***REQUEST_URL***", url)
         response = requests.post(url + "api/interpreter", data=overpass_query)
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))

From e69d80c48d597c5407c64d40b669e61858e7a26e Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 17:22:25 -0600
Subject: [PATCH 10/21] Prints + more db cleanup

Adding some prints to see if I can pinpoint the issue.

Also removing more unnecessary fluff from setup script.
---
 emission/individual_tests/setup_overpass_tests.sh        | 3 ---
 emission/net/ext_service/transit_matching/match_stops.py | 7 ++++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/emission/individual_tests/setup_overpass_tests.sh b/emission/individual_tests/setup_overpass_tests.sh
index 0d5b21474..f56efdc29 100644
--- a/emission/individual_tests/setup_overpass_tests.sh
+++ b/emission/individual_tests/setup_overpass_tests.sh
@@ -1,10 +1,7 @@
 #Set up the testing environment
 # Using an automated install 
-cd /src/e-mission-server
 
-#set database URL using environment variable
 echo ${DB_HOST}
-cat conf/storage/db.conf
 
 echo "Setting up conda..."
 source setup/setup_conda.sh Linux-x86_64
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 004dfd486..7cb505405 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -26,8 +26,8 @@
 
 def make_request_and_catch(overpass_query):
     try:
-        print("***REQUEST_URL***", url)
         response = requests.post(url + "api/interpreter", data=overpass_query)
+        print("*********RESPONSE:" , response.text)
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
         time.sleep(10)
@@ -73,6 +73,7 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
     logging.debug("bbox_string = %s" % bbox_string)
     overpass_public_transit_query_template = query_string
     overpass_query = overpass_public_transit_query_template.format(bbox=bbox_string)
+    print("*************QUERY:", overpass_query)
     call_return = RETRY
     retry_count = 0
     while call_return == RETRY:
@@ -86,6 +87,7 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
 
     logging.info(f"after all retries, retry_count = {retry_count}, call_return = {'RETRY' if call_return == RETRY else len(call_return)}...")
     all_results = call_return
+    print("**********ALL_RESULTS: ", all_results)
 
     relations = [ad.AttrDict(r) for r in all_results if r["type"] == "relation" and r["tags"]["type"] == "route"]
     logging.debug("Found %d relations with ids %s" % (len(relations), [r["id"] for r in relations]))
@@ -108,6 +110,7 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
             rel_nodes_ids = rel_map[relation["id"]]
             if stop.id in rel_nodes_ids:
                 stop["routes"].append({"id": relation["id"], "tags": relation["tags"]})
+    print("***********TRANSIT STOPS: ", stops)
     return stops
 
 # https://gis.stackexchange.com/a/19761
@@ -122,6 +125,7 @@ def get_stops_near(loc, distance_in_meters):
     lon = loc[COORDS][0]
     lat = loc[COORDS][1]
     stops = get_public_transit_stops(lat - bbox_delta, lon - bbox_delta, lat + bbox_delta, lon + bbox_delta)
+    print("********STOPS NEAR!: ", stops)
     logging.debug("Found %d stops" % len(stops))
     for i, stop in enumerate(stops):
         logging.debug("STOP %d: %s" % (i, stop))
@@ -138,6 +142,7 @@ def get_predicted_transit_mode(start_stops, end_stops):
     p_end_routes = list(itertools.chain.from_iterable([extract_routes(s) for s in end_stops]))
 
     rel_id_matches = get_rel_id_match(p_start_routes, p_end_routes)
+    print("**********ID_MATCHES: ", rel_id_matches)
     logging.debug("len(start_routes) = %d, len(end_routes) = %d, len(rel_id_matches) = %d" %
         (len(p_start_routes), len(p_end_routes), len(rel_id_matches)))
     if len(rel_id_matches) > 0:

From b1f7d4feb7e9f7bd5d338ded9649de6a56f9c7cd Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 17:34:22 -0600
Subject: [PATCH 11/21] Printing request url

---
 emission/net/ext_service/transit_matching/match_stops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 7cb505405..a87e15623 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -27,7 +27,7 @@
 def make_request_and_catch(overpass_query):
     try:
         response = requests.post(url + "api/interpreter", data=overpass_query)
-        print("*********RESPONSE:" , response.text)
+        print("*********RESPONSE:" , response.request.url)
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
         time.sleep(10)

From 8186a2d297118204327e2d930b2e4676d462efa4 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Fri, 14 Jun 2024 17:46:05 -0600
Subject: [PATCH 12/21] logging

printing some logs to try to understand what's happening
---
 emission/individual_tests/TestOverpass.py                | 3 +++
 emission/net/ext_service/transit_matching/match_stops.py | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
index 7cffb197e..ff36ac91d 100644
--- a/emission/individual_tests/TestOverpass.py
+++ b/emission/individual_tests/TestOverpass.py
@@ -9,6 +9,7 @@
 import os
 import requests
 import emission.net.ext_service.transit_matching.match_stops as enetm
+import logging
 
 #Set up query
 OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
@@ -22,6 +23,8 @@
 
 class OverpassTest(unittest.TestCase):
     def setUp(self):
+        loglevel = logging.DEBUG
+        logging.basicConfig(level=loglevel)
         sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
         call_base = 'api/interpreter?data='
         self.de_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index a87e15623..778c4beea 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -28,6 +28,8 @@ def make_request_and_catch(overpass_query):
     try:
         response = requests.post(url + "api/interpreter", data=overpass_query)
         print("*********RESPONSE:" , response.request.url)
+        ("*********RESPONSE header:" , response.request.headers)
+        
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
         time.sleep(10)

From 9d7b2f5297c23f1b849a48706c1d57b3ec08a009 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 12:06:01 -0600
Subject: [PATCH 13/21] http --> https

Call isn't going through and I think it's because i'm using http instead of https. This might be a security restriction with GitHub Actions.
---
 emission/individual_tests/TestOverpass.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
index ff36ac91d..4ab6144fe 100644
--- a/emission/individual_tests/TestOverpass.py
+++ b/emission/individual_tests/TestOverpass.py
@@ -28,7 +28,7 @@ def setUp(self):
         sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
         call_base = 'api/interpreter?data='
         self.de_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
-        self.gfbk_url_base = 'http://overpass.geofabrik.de/' + OVERPASS_KEY + '/' + call_base + sample_data
+        self.gfbk_url_base = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/' + call_base + sample_data
 
     def test_overpass(self):
         r_gfbk = requests.get(self.gfbk_url_base)

From ced8b9511304e13d6388f10c38d8b08f464326f7 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 12:09:19 -0600
Subject: [PATCH 14/21] http --> https (in match_stops.py too)

Forgot to change this in the match stops module, too.
---
 emission/net/ext_service/transit_matching/match_stops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 778c4beea..74f580d71 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -8,7 +8,7 @@
 
 try:
     OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
-    url = 'http://overpass.geofabrik.de/' + OVERPASS_KEY + '/'
+    url = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/'
     print("overpass configured")
 except:
     print("overpass not configured, falling back to default overleaf.de")

From f70148268679f57b77db98b4a39c66607553f67e Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 12:24:43 -0600
Subject: [PATCH 15/21] import future

Getting `module not found` error with __future__, so adding it as a direct import to see if that fixes the error.

Also adding more specificity to the echo in the .yml file
---
 .github/workflows/test-overpass.yml | 2 +-
 bin/deploy/push_conf.py             | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index fd64d8fed..367da32ba 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -26,6 +26,6 @@ jobs:
       env:
         OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
       run: |
-        echo Testing!
+        echo Testing overpass!
         chmod +x emission/individual_tests/setup_overpass_tests.sh
         ./emission/individual_tests/setup_overpass_tests.sh
diff --git a/bin/deploy/push_conf.py b/bin/deploy/push_conf.py
index 88accddaa..d7dc004c1 100644
--- a/bin/deploy/push_conf.py
+++ b/bin/deploy/push_conf.py
@@ -1,3 +1,4 @@
+import __future__
 from __future__ import unicode_literals
 from __future__ import print_function
 from __future__ import division

From 06e0a40205cc86a1a32421d263d4b5d38abd29e5 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 12:53:37 -0600
Subject: [PATCH 16/21] revert push_conf + updated api key

adding import just caused another error. Testing now that I updated the api key.
---
 bin/deploy/push_conf.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/deploy/push_conf.py b/bin/deploy/push_conf.py
index d7dc004c1..88accddaa 100644
--- a/bin/deploy/push_conf.py
+++ b/bin/deploy/push_conf.py
@@ -1,4 +1,3 @@
-import __future__
 from __future__ import unicode_literals
 from __future__ import print_function
 from __future__ import division

From 5139a39ee213711e913a42d230bedec6c6c73b8e Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 13:06:16 -0600
Subject: [PATCH 17/21] Cleanup 1

Increasing specificity for prints; removing unnecessary prints; renaming certain variables so they're more meaningful.
---
 emission/individual_tests/TestOverpass.py     | 19 +++++++++----------
 .../transit_matching/match_stops.py           |  9 +--------
 2 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
index 4ab6144fe..1e8d524c2 100644
--- a/emission/individual_tests/TestOverpass.py
+++ b/emission/individual_tests/TestOverpass.py
@@ -23,28 +23,28 @@
 
 class OverpassTest(unittest.TestCase):
     def setUp(self):
-        loglevel = logging.DEBUG
-        logging.basicConfig(level=loglevel)
+        # Un-comment the two lines below to print debug logs.
+        # loglevel = logging.DEBUG
+        # logging.basicConfig(level=loglevel)
         sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
         call_base = 'api/interpreter?data='
-        self.de_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
+        self.public_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
         self.gfbk_url_base = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/' + call_base + sample_data
 
     def test_overpass(self):
         r_gfbk = requests.get(self.gfbk_url_base)
-        r_de = requests.get(self.de_url_base)
+        r_public = requests.get(self.public_url_base)
         
-        if r_gfbk.status_code == 200 and r_de.status_code == 200:
+        if r_gfbk.status_code == 200 and r_public.status_code == 200:
             print("requests successful!")
-            r_gfbk_len, r_de_len = len(r_gfbk.json()), len(r_de.json())
-            self.assertEqual(r_gfbk_len, r_de_len)
+            r_gfbk_len, r_public_len = len(r_gfbk.json()), len(r_public.json())
+            self.assertEqual(r_gfbk_len, r_public_len)
         else:
-            print("status_gfbk", r_gfbk.status_code, type(r_gfbk.status_code), "status_de", r_de.status_code)
+            print("status_gfbk", r_gfbk.status_code, type(r_gfbk.status_code), "status_public", r_public.status_code)
 
     #Test utilizes the functions get_stops_near, get_public_transit_stops, and make_request_and_catch.  
     def test_get_stops_near(self):
         actual_result = enetm.get_stops_near(loc1, 150.0)[0]['routes'][0]['tags']
-        print("ACTUAL_RESULT:", actual_result, type(actual_result))
         expected_result = {'from': 'National Renewable Energy Lab', 'name': 'RTD Route 125: Red Rocks College', 'network': 'RTD', 'network:wikidata': 'Q7309183', 'network:wikipedia': 'en:Regional Transportation District', 'operator': 'Regional Transportation District', 'public_transport:version': '1', 'ref': '125', 'route': 'bus', 'to': 'Red Rocks College', 'type': 'route'}
         self.assertEqual(expected_result, actual_result)
    
@@ -54,7 +54,6 @@ def test_get_predicted_transit_mode(self):
         stop1 = enetm.get_stops_near(loc2, 400.0)
         stop2 = enetm.get_stops_near(loc3, 400.0)
         actual_result = enetm.get_predicted_transit_mode(stop1, stop2)
-        print("ACTUAL TRANSIT MODE: ", actual_result)
         expected_result = ['train', 'train']
         self.assertEqual(actual_result, expected_result)
 
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 74f580d71..a39afd681 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -11,7 +11,7 @@
     url = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/'
     print("overpass configured")
 except:
-    print("overpass not configured, falling back to default overleaf.de")
+    print("overpass not configured, falling back to public overpass api")
     url = "https://lz4.overpass-api.de/"
 
 try:
@@ -27,8 +27,6 @@
 def make_request_and_catch(overpass_query):
     try:
         response = requests.post(url + "api/interpreter", data=overpass_query)
-        print("*********RESPONSE:" , response.request.url)
-        ("*********RESPONSE header:" , response.request.headers)
         
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
@@ -75,7 +73,6 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
     logging.debug("bbox_string = %s" % bbox_string)
     overpass_public_transit_query_template = query_string
     overpass_query = overpass_public_transit_query_template.format(bbox=bbox_string)
-    print("*************QUERY:", overpass_query)
     call_return = RETRY
     retry_count = 0
     while call_return == RETRY:
@@ -89,7 +86,6 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
 
     logging.info(f"after all retries, retry_count = {retry_count}, call_return = {'RETRY' if call_return == RETRY else len(call_return)}...")
     all_results = call_return
-    print("**********ALL_RESULTS: ", all_results)
 
     relations = [ad.AttrDict(r) for r in all_results if r["type"] == "relation" and r["tags"]["type"] == "route"]
     logging.debug("Found %d relations with ids %s" % (len(relations), [r["id"] for r in relations]))
@@ -112,7 +108,6 @@ def get_public_transit_stops(min_lat, min_lon, max_lat, max_lon):
             rel_nodes_ids = rel_map[relation["id"]]
             if stop.id in rel_nodes_ids:
                 stop["routes"].append({"id": relation["id"], "tags": relation["tags"]})
-    print("***********TRANSIT STOPS: ", stops)
     return stops
 
 # https://gis.stackexchange.com/a/19761
@@ -127,7 +122,6 @@ def get_stops_near(loc, distance_in_meters):
     lon = loc[COORDS][0]
     lat = loc[COORDS][1]
     stops = get_public_transit_stops(lat - bbox_delta, lon - bbox_delta, lat + bbox_delta, lon + bbox_delta)
-    print("********STOPS NEAR!: ", stops)
     logging.debug("Found %d stops" % len(stops))
     for i, stop in enumerate(stops):
         logging.debug("STOP %d: %s" % (i, stop))
@@ -144,7 +138,6 @@ def get_predicted_transit_mode(start_stops, end_stops):
     p_end_routes = list(itertools.chain.from_iterable([extract_routes(s) for s in end_stops]))
 
     rel_id_matches = get_rel_id_match(p_start_routes, p_end_routes)
-    print("**********ID_MATCHES: ", rel_id_matches)
     logging.debug("len(start_routes) = %d, len(end_routes) = %d, len(rel_id_matches) = %d" %
         (len(p_start_routes), len(p_end_routes), len(rel_id_matches)))
     if len(rel_id_matches) > 0:

From 8a93991522f42ae758128af94c0faa0a2d067e14 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 13:11:43 -0600
Subject: [PATCH 18/21] Cleanup 2

Basic cleanup + putting action on a schedule rather than a push.
---
 .github/workflows/test-overpass.yml                      | 9 +++------
 emission/net/ext_service/transit_matching/match_stops.py | 3 +--
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index 367da32ba..c81580a43 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -3,13 +3,10 @@ name: overpass-test
 # Controls when the action will run. Triggers the workflow on push or pull request
 # events but only for the master branch
 on:
-  push:
-    branches:
-      - overpass
-  # schedule:
+  schedule:
 
-  #   # Run every Sunday at 6:05 am
-  #   - cron:  '5 6 * * 0'
+    # Run every Sunday at 6:05 am
+    - cron:  '5 6 * * 0'
 jobs:
   build:
     runs-on: ubuntu-latest
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index a39afd681..eabb083b6 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -26,8 +26,7 @@
 
 def make_request_and_catch(overpass_query):
     try:
-        response = requests.post(url + "api/interpreter", data=overpass_query)
-        
+        response = requests.post(url + "api/interpreter", data=overpass_query)   
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
         time.sleep(10)

From 1a6da67216fce7fc1370c2163d58009d3f21118a Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Mon, 17 Jun 2024 13:32:22 -0600
Subject: [PATCH 19/21] Cleanup 3

Removing superfluous spaces.
---
 emission/net/ext_service/transit_matching/match_stops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index eabb083b6..3ad38b018 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -26,7 +26,7 @@
 
 def make_request_and_catch(overpass_query):
     try:
-        response = requests.post(url + "api/interpreter", data=overpass_query)   
+        response = requests.post(url + "api/interpreter", data=overpass_query)
     except requests.exceptions.ChunkedEncodingError as e:
         logging.info("ChunkedEncodingError while creating request %s" % (e))
         time.sleep(10)

From 243742f50324b6469840b8e6a7874f9deead0947 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Tue, 25 Jun 2024 12:09:29 -0600
Subject: [PATCH 20/21] Rename setup_overpass_tests.sh to
 setup_and_test_overpass.sh

---
 .../{setup_overpass_tests.sh => setup_and_test_overpass.sh}       | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename emission/individual_tests/{setup_overpass_tests.sh => setup_and_test_overpass.sh} (100%)

diff --git a/emission/individual_tests/setup_overpass_tests.sh b/emission/individual_tests/setup_and_test_overpass.sh
similarity index 100%
rename from emission/individual_tests/setup_overpass_tests.sh
rename to emission/individual_tests/setup_and_test_overpass.sh

From 02f8e542a4b57e3429357b937091d018605749a7 Mon Sep 17 00:00:00 2001
From: Natalie Schultz <90212258+nataliejschultz@users.noreply.github.com>
Date: Tue, 25 Jun 2024 12:16:18 -0600
Subject: [PATCH 21/21] OVERPASS_KEY --> GEOFABRIK_OVERPASS_KEY

Changing environment variable name and adding functionality for the new setup script name.
---
 .github/workflows/test-overpass.yml                      | 6 +++---
 emission/individual_tests/TestOverpass.py                | 4 ++--
 emission/net/ext_service/transit_matching/match_stops.py | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test-overpass.yml b/.github/workflows/test-overpass.yml
index c81580a43..934075e4f 100644
--- a/.github/workflows/test-overpass.yml
+++ b/.github/workflows/test-overpass.yml
@@ -21,8 +21,8 @@ jobs:
       # Runs TestOverpass.py to check for API functionality
     - name: Test Overpass
       env:
-        OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
+        GEOFABRIK_OVERPASS_KEY: '${{ secrets.OVERPASS_API }}'
       run: |
         echo Testing overpass!
-        chmod +x emission/individual_tests/setup_overpass_tests.sh
-        ./emission/individual_tests/setup_overpass_tests.sh
+        chmod +x emission/individual_tests/setup_and_test_overpass.sh
+        ./emission/individual_tests/setup_and_test_overpass.sh
diff --git a/emission/individual_tests/TestOverpass.py b/emission/individual_tests/TestOverpass.py
index 1e8d524c2..aedb64390 100644
--- a/emission/individual_tests/TestOverpass.py
+++ b/emission/individual_tests/TestOverpass.py
@@ -12,7 +12,7 @@
 import logging
 
 #Set up query
-OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
+GEOFABRIK_OVERPASS_KEY = os.environ.get("GEOFABRIK_OVERPASS_KEY")
 
 #Sample loc1 = NREL East Gate
 loc1 = {'coordinates': [-105.16844103184974, 39.740428870224605]}
@@ -29,7 +29,7 @@ def setUp(self):
         sample_data = '[out:json][bbox];way[amenity=parking];out;&bbox=-122.1111238,37.4142118,-122.1055791,37.4187945'
         call_base = 'api/interpreter?data='
         self.public_url_base = 'https://lz4.overpass-api.de/'+ call_base + sample_data
-        self.gfbk_url_base = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/' + call_base + sample_data
+        self.gfbk_url_base = 'https://overpass.geofabrik.de/' + GEOFABRIK_OVERPASS_KEY + '/' + call_base + sample_data
 
     def test_overpass(self):
         r_gfbk = requests.get(self.gfbk_url_base)
diff --git a/emission/net/ext_service/transit_matching/match_stops.py b/emission/net/ext_service/transit_matching/match_stops.py
index 3ad38b018..afa5d6abd 100644
--- a/emission/net/ext_service/transit_matching/match_stops.py
+++ b/emission/net/ext_service/transit_matching/match_stops.py
@@ -7,8 +7,8 @@
 import time
 
 try:
-    OVERPASS_KEY = os.environ.get("OVERPASS_KEY")
-    url = 'https://overpass.geofabrik.de/' + OVERPASS_KEY + '/'
+    GEOFABRIK_OVERPASS_KEY = os.environ.get("GEOFABRIK_OVERPASS_KEY")
+    url = 'https://overpass.geofabrik.de/' + GEOFABRIK_OVERPASS_KEY + '/'
     print("overpass configured")
 except:
     print("overpass not configured, falling back to public overpass api")