Merge pull request #62 from GIScience/development

Update dependencies in readme, better handling of invalid json responses, improved logging.
GIScience · Jun 4, 2021 · 67e6405 · 67e6405
2 parents a4ea9ea + c480fd3
commit 67e6405
Show file tree

Hide file tree

Showing 11 changed files with 297 additions and 20 deletions.
diff --git a/.gitignore b/.gitignore
@@ -134,3 +134,7 @@ dmypy.json
 
 # Setup.py as it will be generated on the fly through dephell
 setup.py
+
+
+# project files
+**/ohsome_log/
diff --git a/README.md b/README.md
@@ -6,16 +6,36 @@ The *ohsome-py* package helps you extract and analyse OpenStreetMap history data
 
 The ohsome API provides various endpoints for [data aggregation](https://api.ohsome.org/v1/swagger-ui.html?urls.primaryName=Data%20Aggregation), [data extraction](https://api.ohsome.org/v1/swagger-ui.html?urls.primaryName=dataExtraction) and [contributions](https://api.ohsome.org/v1/swagger-ui.html?urls.primaryName=Contributions) to analyse the history of OSM data. Take a look at the [documentation of the ohsome API](https://docs.ohsome.org/ohsome-api/stable) or go through the [Tutorial](https://github.com/GIScience/ohsome-py/blob/master/notebooks/Tutorial.ipynb) to get started on how to use *ohsome-py*.
 
-
 ## Installation
 
-*ohsome-py* requires Python >= 3.6. The easiest way to install *ohsome-py* is using pip:
+*ohsome-py* requires
+
+* Python >= 3.6
+* geopandas >= 0.9.0
+* pyproj >= 3.0.0
+* requests >= 2.25.1
+
+### Using pip
+
+You can install *ohsome-py* using pip, which will also install all dependencies.
 
 ```
 $ pip install ohsome
 ```
 
-If you want to run the Juypter Notebook [Tutorial](https://github.com/GIScience/ohsome-py/blob/master/notebooks/Tutorial.ipynb) you also need to install `jupyter` and `matplotlib`:
+### Using Anaconda
+
+*ohsome-py* is not available through Anaconda yet. So if you are using Anaconda, create a new anaconda environment and install the required dependencies before installing *ohsome-py* using pip. Please note that there might be issues when [using pip within anaconda](https://www.anaconda.com/blog/using-pip-in-a-conda-environment). To avoid issues make sure to install everythin in a new conda environment.
+
+```
+$ conda create -n ohsome python=3.8 geopandas>=0.9.0 requests>=2.25.1
+$ conda activate ohsome
+$ pip install ohsome --no-deps
+```
+
+### Dependencies for Jupyter Notebooks
+
+If you want to run the Juypter Notebook [Tutorial](https://github.com/GIScience/ohsome-py/blob/master/notebooks/Tutorial.ipynb) you also need to install `jupyter` and `matplotlib` e.g.
 
 ```
 $ pip install jupyter matplotlib
@@ -126,7 +146,19 @@ time = pandas.date_range("2018-01-01", periods=3, freq="M")
 
 If you want to contribute to this project, please fork the repository or create a new branch containing your changes.
 
-**Install the pre-commit hooks** in our local git repo before commiting to ensure homogenous code style.
+**Install dependencies for development**
+
+All of these dependecies can be installed using pip. If you are using anaconda, you need to install the packages through conda-forge.
+
+* pytest = ^6.2.2
+* pytest-cov = >=2.0.0
+* pre-commit = >=2.1.1
+* black = ^20.8b1
+* pytest-random-order = ^1.0.4 (not available through anaconda)
+* yaspin = <1.4.1
+* tox = ^3.23.0
+
+**Install the pre-commit hooks** in our local git repo before committing to ensure homogenous code style.
 
 ```
 $ pre-commit install

diff --git a/ohsome/clients.py b/ohsome/clients.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 """OhsomeClient classes to build and handle requests to ohsome API"""
+import urllib
 
 import requests
 from ohsome import OhsomeException, OhsomeResponse
@@ -285,7 +286,7 @@ def _handle_request(self):
                 error_code=440,
             )
         except ValueError:
-            error_code, message = extract_error_message_from_invalid_json(response)
+            error_code, message = extract_error_message_from_invalid_json(response.text)
             ohsome_exception = OhsomeException(
                 message=message,
                 url=self._url,
@@ -334,9 +335,9 @@ def _construct_resource_url(self, endpoint=None):
         :return:
         """
         if endpoint:
-            self._url = self._base_api_url + "/" + endpoint.strip("/")
+            self._url = urllib.parse.urljoin(self._base_api_url, endpoint.strip("/"))
         else:
-            self._url = self._base_api_url + "/".join(self._cache)
+            self._url = urllib.parse.urljoin(self._base_api_url, "/".join(self._cache))
 
     def _(self, name):
         # Enables method chaining

diff --git a/ohsome/exceptions.py b/ohsome/exceptions.py
@@ -29,14 +29,25 @@ def log(self, log_dir):
         Logs OhsomeException
         :return:
         """
-        log_file_name = (
-            f"ohsome_{self.error_code}_{dt.datetime.now().strftime('%Y-%m-%dT%H%M%S')}"
-        )
+        log_file_name = f"ohsome_{dt.datetime.now().strftime('%Y-%m-%dT%H%M%S')}"
         self.log_bpolys(log_dir, log_file_name)
+        self.log_parameter(log_dir, log_file_name)
         self.log_response(log_dir, log_file_name)
         # self.log_query(log_dir, log_file_name)
 
     def log_response(self, log_dir, log_file_name):
+        """
+        Log raw response. This may duplicate much data but is helpful for debugging to know the exact raw answer by the
+        ohsome-api.
+        """
+        log_file = os.path.join(
+            log_dir,
+            f"{log_file_name}_raw.txt",
+        )
+        with open(log_file, "w") as dst:
+            dst.write(self.response.text)
+
+    def log_parameter(self, log_dir, log_file_name):
         """
         Log query parameters to file
         :param log_dir:

diff --git a/ohsome/helper.py b/ohsome/helper.py
@@ -10,6 +10,8 @@
 
 from ohsome import OhsomeException
 
+import re
+
 
 def format_time(params):
     """
@@ -175,19 +177,50 @@ def find_groupby_names(url):
     return [name.strip("/") for name in url.split("groupBy")[1:]]
 
 
-def extract_error_message_from_invalid_json(response):
+def extract_error_message_from_invalid_json(responsetext):
     """
     Extract error code and error message from invalid json returned from ohsome API
     Otherwise throws OhsomeException.
     :param response:
     :return:
 
     """
-    start_message = response.text.find("message") + 12
-    end_message = response.text.find("requestUrl") - 6
-    message = response.text[start_message:end_message]
-    start_error_code = response.text.find("status") + 10
-    end_error_code = response.text.find("message") - 5
-    error_code = response.text[start_error_code:end_error_code]
+    # responsetext = response.text
+
+    message = "A broken response has been received"
+
+    m = re.search('"message" : "(.*)"', responsetext)
+    if m:
+        message += ": " + m.group(1)
+
+    m = re.search('"error" : "(.*)"', responsetext)
+    if m:
+        message += "; " + m.group(0)
+
+    m = re.search('"timestamp" : "(.*)"', responsetext)
+    if m:
+        message += "; " + m.group(0)
+
+    m = re.search('"path" : "(.*)"', responsetext)
+    if m:
+        message += "; " + m.group(0)
+
+    m = re.search('"requestUrl" : "(.*)"', responsetext)
+    if m:
+        message += "; " + m.group(0)
+
+    m = re.search(r'"status" : (\d+|".*")', responsetext)
+    if m:
+        status = m.group(1)
+        message += "; " + m.group(0)
+    else:
+        status = None
+
+    if status and status.isdigit() and not (int(status) == 200):
+        error_code = int(status)
+    elif "OutOfMemoryError" in message:
+        error_code = 507
+    else:
+        error_code = 500
 
     return error_code, message
diff --git a/ohsome/test/data/invalid_response.txt b/ohsome/test/data/invalid_response.txt
@@ -0,0 +1,48 @@
+{
+  "attribution" : {
+    "url" : "https://ohsome.org/copyrights",
+    "text" : "© OpenStreetMap contributors"
+  },
+  "apiVersion" : "0.9",
+  "type" : "FeatureCollection",
+  "features" : [{
+    "type" : "Feature",
+    "geometry" : {
+      "type" : "Polygon",
+      "coordinates" : [
+        [
+          [
+            13.644420799999999,
+            50.9673861
+          ],
+          [
+            13.6446461,
+            50.967762699999994
+          ],
+          [
+            13.6435705,
+            50.9680009
+          ],
+          [
+            13.6433485,
+            50.9676301
+          ],
+          [
+            13.644420799999999,
+            50.9673861
+          ]
+        ]
+      ]
+    },
+    "properties" : {
+      "@osmId" : "way/437313075",
+      "@snapshotTimestamp" : "2019-12-10T00:00:00Z",
+      "leisure" : "park"
+    }
+  }{
+  "timestamp" : "2020-05-19T07:07:25.356+0000",
+  "status" : 200,
+  "error" : "OK",
+  "message" : "java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.sql.SQLTransientConnectionException: HikariPool-1 - Connection is not available, request timed out after 30000ms.",
+  "path" : "/elements/geometry"
+}
diff --git a/ohsome/test/data/invalid_response_customCode.txt b/ohsome/test/data/invalid_response_customCode.txt
@@ -0,0 +1,12 @@
+  "attribution" : {
+    "url" : "https://ohsome.org/copyrights",
+    "text" : "© OpenStreetMap contributors"
+  },
+  "apiVersion" : "1.3.2",
+  "type" : "FeatureCollection",
+  "features" : [{
+  "timestamp" : "2021-06-02T10:07:46.438591",
+  "status" : 413,
+  "message" : "The given query is too large in respect to the given timeout. Please use a smaller region and/or coarser time period.",
+  "requestUrl" : "http://localhost:8080/elements/geometry"
+}
diff --git a/ohsome/test/data/invalid_response_no_message.txt b/ohsome/test/data/invalid_response_no_message.txt
@@ -0,0 +1,8 @@
+, {
+    "type" : "Feature",
+    "geometry" : {
+      "type" : "Polygon",
+      "coordinates" : [
+        [
+          [
+            8.3162482,
diff --git a/ohsome/test/data/invalid_response_outOfMemory.txt b/ohsome/test/data/invalid_response_outOfMemory.txt
@@ -0,0 +1,46 @@
+{
+  "attribution" : {
+    "url" : "https://ohsome.org/copyrights",
+    "text" : "© OpenStreetMap contributors"
+  },
+  "apiVersion" : "1.4.1",
+  "type" : "FeatureCollection",
+  "features" : [{
+    "type" : "Feature",
+    "geometry" : {
+      "type" : "Polygon",
+      "coordinates" : [
+        [
+          [
+            8.7833329,
+            49.5470456
+          ],
+          [
+            8.7836152,
+            49.5466073
+          ],
+          [
+            8.7851933,
+            49.5470193
+          ],
+          [
+            8.785043,
+            49.5473679
+          ],
+          [
+            8.7849303,
+            49.5473947
+          ],
+          [
+            8.7833329,
+            49.5470456
+          ]
+        ]
+      ]
+    }{
+  "timestamp" : "2021-06-01T11:38:52.821+0000",
+  "status" : 200,
+  "error" : "OK",
+  "message" : "java.lang.OutOfMemoryError",
+  "path" : "/elements/geometry"
+}
diff --git a/ohsome/test/test_exceptions.py b/ohsome/test/test_exceptions.py
@@ -110,7 +110,7 @@ def test_enable_logging(custom_client_without_log, tmpdir):
         )
 
     n_log_files_after = len(os.listdir(custom_client_without_log.log_dir))
-    assert n_log_files_before + 1 == n_log_files_after
+    assert n_log_files_before + 2 == n_log_files_after
 
 
 def test_log_bpolys(custom_client_without_log, tmpdir):
@@ -134,7 +134,7 @@ def test_log_bpolys(custom_client_without_log, tmpdir):
             bpolys=bpolys, time=time, filter=fltr, timeout=timeout
         )
     n_log_files_after = len(os.listdir(custom_client_without_log.log_dir))
-    assert n_log_files_before + 2 == n_log_files_after
+    assert n_log_files_before + 3 == n_log_files_after
 
 
 def test_metadata_invalid_baseurl(custom_client_with_wrong_url):