BD-3407 upgrade python version in python-app.yml

.github/workflows/python-app.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python_version: ['3.8']
+        python_version: ['3.12']
         code_command:
         - flake8
         - isort --check-only --diff
@@ -43,10 +43,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Docker build test image
       run: |
-        docker-compose build pytest
+        docker compose build pytest
     - name: Pip list
       run: |
-        docker-compose run pytest python -m pip list
+        docker compose run pytest python -m pip list
     - name: Test
       run: |
-        docker-compose run pytest
+        docker compose run pytest

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Function location_for_hive_table
 - Lineage class to get downstream_tables
 - pii.Producer and pii.Consumer for creating pii removal requests
+- Upgrade to Python 3.10 and above
 
 ### Fixed
 - DeltaTable.isDeltaTable doesn't seem to work with Unity Catalog,

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Niels Lemmens
+Copyright (c) 2024 HusqvarnaGroup
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,9 +1,16 @@
 # Delta utils
 
+## Requirements
+
+- Python 3.10
+- Databricks Runtime Version 13.3 and above
+
+
 ## Installation
 
+Look for the latest version in the [Github Tags section](https://github.com/husqvarnagroup/delta_utils/tags) of this repo.
 ```bash
-pip install git+https://github.com/husqvarnagroup/delta_utils.git
+pip install https://github.com/husqvarnagroup/delta_utils/archive/0.4.0.zip
 ```
 
 ### Installation on a cluster in Databricks
@@ -12,7 +19,7 @@ pip install git+https://github.com/husqvarnagroup/delta_utils.git
 - Go to Libraries
 - Click Install new
 - Select PyPI
-- As the package, enter "git+https://github.com/husqvarnagroup/delta_utils.git"
+- As the package, enter "https://github.com/husqvarnagroup/delta_utils/archive/0.4.0.zip"
 - Press Install
 
 ## Development

delta_utils/core.py

@@ -36,10 +36,7 @@ def read_change_feed(spark: SparkSession, path: str, **kwargs) -> DataFrame:
     except AnalysisException as e:
         error_msg = str(e)
         print(error_msg)
-        if (
-            error_msg.startswith("The provided timestamp")
-            and "is after the latest version available to this" in error_msg
-        ):
+        if "DELTA_TIMESTAMP_GREATER_THAN_COMMIT" in error_msg:
             raise NoNewDataException(error_msg)
         else:
             raise e

delta_utils/fileregistry.py

@@ -3,6 +3,7 @@
 This feature is deprecated. Databricks Unity Catalog disables s3 access, we recommend you to use
 (Databricks Auto Loader)[https://docs.databricks.com/ingestion/auto-loader/options.html] instead
 """
+
 import uuid
 from contextlib import contextmanager
 from dataclasses import dataclass

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
     pytest:
         build:

pyproject.toml

@@ -9,15 +9,15 @@ documentation = "https://delta-utils.readthedocs.io/en/latest/"
 license = "MIT"
 
 [tool.poetry.dependencies]
-python = "^3.8"
-boto3 = "^1.28.51"
+python = "^3.10"
+boto3 = "^1.34"
 reverse_geocoder = "*"
 iso3166 = "*"
 requests = "*"
 
 [tool.poetry.dev-dependencies]
 delta-spark = "*"
-pyspark = "<3.4"
+pyspark = "<4"
 mkdocs-material = "*"
 mkdocstrings-python = "*"
 black = "*"

tests/conftest.py

@@ -3,7 +3,7 @@
 import boto3
 import pytest
 from delta import configure_spark_with_delta_pip
-from moto import mock_s3  # type: ignore
+from moto import mock_aws  # type: ignore
 from pyspark.sql import SparkSession
 
 from delta_utils import pii
@@ -27,7 +27,7 @@ def spark():
 
 @pytest.fixture(scope="function")
 def mocked_s3_bucket_name():
-    with mock_s3():
+    with mock_aws():
         conn = boto3.resource("s3", region_name="us-east-1")
         conn.create_bucket(Bucket="mybucket")
         yield "mybucket"

tests/test_ssm.py

@@ -1,4 +1,5 @@
 """Unit tests of the delta_utils.ssm module"""
+
 from delta_utils import ssm