From 3afd602a9f76d8239c399be317f4a221adef44f9 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Fri, 20 Dec 2024 00:01:39 +0100 Subject: [PATCH 01/33] move aws tests to subdirectory --- tests/{ => aws}/test_aws.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ => aws}/test_aws.py (100%) diff --git a/tests/test_aws.py b/tests/aws/test_aws.py similarity index 100% rename from tests/test_aws.py rename to tests/aws/test_aws.py From 7c33c5be877d72f71b3ab8fe8b98731bd6689ff0 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sat, 21 Dec 2024 21:56:04 +0100 Subject: [PATCH 02/33] end to end test for aws.s3 upload --- src/unicloud/aws/aws.py | 18 +++++++++----- tests/aws/test_aws.py | 53 ++++++++++++++++++----------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 02ef86b..47f48ab 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -1,6 +1,7 @@ """S3 Cloud Storage.""" from typing import Optional +import traceback import boto3 @@ -35,14 +36,14 @@ def __init__( @property def client(self): - """client.""" + """AWS S3 Client.""" return self._client def create_client(self): """Create and returns an AWS S3 client instance. initializing the AWS S3 client, passing credentials directly is one option. Another approach is to use AWS - IAM roles for EC2 instances or to configure the AWS CLI with aws configure, which sets up the credentials + IAM roles for EC2 instances or to configure the AWS CLI with aws configure, which sets up the credentials' file used by boto3. This can be a more secure and manageable way to handle credentials, especially in production environments. @@ -55,19 +56,24 @@ def create_client(self): aws_secret_access_key=self.aws_secret_access_key, ) - def upload(self, path: str, bucket_path: str): + def upload(self, local_path: str, bucket_path: str): """Upload a file to S3. Parameters ---------- - path: [str] + local_path: [str] The path to the file to upload. bucket_path: [str] The bucket_path in the format "bucket_name/object_name". """ bucket_name, object_name = bucket_path.split("/", 1) - self.client.upload_file(path, bucket_name, object_name) - print(f"File {path} uploaded to {bucket_path}.") + try: + self.client.upload_file(local_path, bucket_name, object_name) + except Exception as e: + print("Error uploading file to S3:") + print(traceback.format_exc()) + raise e + print(f"File {local_path} uploaded to {bucket_path}.") def download(self, bucket_path: str, file_path: str): """Download a file from S3. diff --git a/tests/aws/test_aws.py b/tests/aws/test_aws.py index 060a300..4cb7bf9 100644 --- a/tests/aws/test_aws.py +++ b/tests/aws/test_aws.py @@ -1,4 +1,5 @@ """This module contains tests for the S3 class in unicloud/aws.py.""" +from pathlib import Path import boto3 import pytest @@ -7,6 +8,21 @@ from unicloud.aws.aws import S3 MY_TEST_BUCKET = "test-bucket" +MY_TEST_BUCKET = "testing-unicloud" +AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") +AWS_SECRET_ACCESS_KEY = os.getenv("aws_secret_access_key") +REGION = "eu-central-1" + + +@pytest.fixture +def boto_client() -> boto3.client: + return boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=REGION, + ) + class TestS3: @@ -60,41 +76,18 @@ def test_download_data(self, test_file: str, test_file_content: str): assert f.read() == test_file_content -@pytest.fixture -def aws_credentials(): - """Mocked AWS Credentials for moto.""" - return { - "aws_access_key_id": "testing", - "aws_secret_access_key": "testing", - "region_name": "us-east-1", - } - - -@pytest.fixture -def s3_client(aws_credentials): - """Create an S3 client for testing.""" - with mock_aws(): - boto3.client("s3", region_name=aws_credentials["region_name"]).create_bucket( - Bucket="my-test-bucket" - ) - yield S3(**aws_credentials) - - class TestS3E2E: """End-to-end tests for the S3 class.""" - def test_s3_upload(self, s3_client, tmp_path): - """Test file upload to S3.""" - # Create a temporary file to upload - file_path = tmp_path / "test_upload.txt" - file_path.write_text("Hello, world!") - - s3_client.upload(str(file_path), f"{MY_TEST_BUCKET}/test_upload.txt") + s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) + def test_s3_upload(self, test_file: Path, boto_client: boto3.client): + """Test file upload to S3.""" + upload_file_name = "test_upload.txt" + self.s3.upload(test_file, f"{MY_TEST_BUCKET}/{upload_file_name}") # Verify the file exists in S3 - s3 = boto3.client("s3", region_name="us-east-1") - response = s3.list_objects_v2(Bucket=MY_TEST_BUCKET) - assert "test_upload.txt" in [obj["Key"] for obj in response["Contents"]] + response = boto_client.list_objects_v2(Bucket=MY_TEST_BUCKET) + assert upload_file_name in [obj["Key"] for obj in response["Contents"]] def test_s3_download(self, s3_client, tmp_path): """Test file download from S3.""" From e160fe61ca01d1f5bbc4e397bb4af737af79478f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sat, 21 Dec 2024 22:13:21 +0100 Subject: [PATCH 03/33] end to end test for aws.s3 download --- src/unicloud/aws/aws.py | 6 ++++-- tests/aws/test_aws.py | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 47f48ab..3358917 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -75,16 +75,18 @@ def upload(self, local_path: str, bucket_path: str): raise e print(f"File {local_path} uploaded to {bucket_path}.") - def download(self, bucket_path: str, file_path: str): + def download(self, bucket_path: str, local_path: str): """Download a file from S3. Parameters ---------- bucket_path: [str] The bucket_path in the format "bucket_name/object_name". - file_path: [str] + local_path: [str] The path to save the downloaded file. """ bucket_name, object_name = bucket_path.split("/", 1) self.client.download_file(bucket_name, object_name, file_path) print(f"File {bucket_path} downloaded to {file_path}.") + self.client.download_file(bucket_name, object_name, local_path) + print(f"File {bucket_path} downloaded to {local_path}.") diff --git a/tests/aws/test_aws.py b/tests/aws/test_aws.py index 4cb7bf9..c03c86d 100644 --- a/tests/aws/test_aws.py +++ b/tests/aws/test_aws.py @@ -1,4 +1,6 @@ """This module contains tests for the S3 class in unicloud/aws.py.""" + +import os from pathlib import Path import boto3 @@ -7,7 +9,6 @@ from unicloud.aws.aws import S3 -MY_TEST_BUCKET = "test-bucket" MY_TEST_BUCKET = "testing-unicloud" AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") AWS_SECRET_ACCESS_KEY = os.getenv("aws_secret_access_key") @@ -80,20 +81,23 @@ class TestS3E2E: """End-to-end tests for the S3 class.""" s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) + file_name = "test_upload.txt" def test_s3_upload(self, test_file: Path, boto_client: boto3.client): """Test file upload to S3.""" - upload_file_name = "test_upload.txt" - self.s3.upload(test_file, f"{MY_TEST_BUCKET}/{upload_file_name}") + + self.s3.upload(test_file, f"{MY_TEST_BUCKET}/{self.file_name}") # Verify the file exists in S3 response = boto_client.list_objects_v2(Bucket=MY_TEST_BUCKET) - assert upload_file_name in [obj["Key"] for obj in response["Contents"]] + assert self.file_name in [obj["Key"] for obj in response["Contents"]] - def test_s3_download(self, s3_client, tmp_path): + def test_s3_download(self, test_file_content: str): """Test file download from S3.""" - # Assuming the file "test_upload.txt" already exists in S3 from the previous test - download_path = tmp_path / "downloaded_test.txt" - s3_client.download(f"{MY_TEST_BUCKET}/test_upload.txt", str(download_path)) + + download_path = Path("tests/data/aws-test-file.txt") + self.s3.download(f"{MY_TEST_BUCKET}/{self.file_name}", download_path) # Verify the file content assert download_path.read_text() == "Hello, world!" + assert download_path.read_text() == test_file_content + os.remove(download_path) From 161e9268a43b027a7f3990f2c5449f50a447c3be Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sat, 21 Dec 2024 23:26:54 +0100 Subject: [PATCH 04/33] create `aws.Bucket` class --- src/unicloud/aws/aws.py | 26 ++++++++++++++++++++++++-- tests/aws/test_aws.py | 9 +++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 3358917..6f68644 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -86,7 +86,29 @@ def download(self, bucket_path: str, local_path: str): The path to save the downloaded file. """ bucket_name, object_name = bucket_path.split("/", 1) - self.client.download_file(bucket_name, object_name, file_path) - print(f"File {bucket_path} downloaded to {file_path}.") self.client.download_file(bucket_name, object_name, local_path) print(f"File {bucket_path} downloaded to {local_path}.") + + def get_bucket(self, bucket_name: str) -> "Bucket": + """Retrieve a bucket object.""" + s3 = boto3.resource( + "s3", + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + region_name=self.region_name, + ) + bucket = s3.Bucket(bucket_name) + return Bucket(bucket) + + +class Bucket: + """S3 Bucket.""" + + def __init__(self, bucket): # :boto3.resources("s3").Bucket + """Initialize the S3 bucket.""" + self._bucket = bucket + + @property + def bucket(self): + """bucket.""" + return self._bucket diff --git a/tests/aws/test_aws.py b/tests/aws/test_aws.py index c03c86d..aa3413b 100644 --- a/tests/aws/test_aws.py +++ b/tests/aws/test_aws.py @@ -7,7 +7,7 @@ import pytest from moto import mock_aws -from unicloud.aws.aws import S3 +from unicloud.aws.aws import S3, Bucket MY_TEST_BUCKET = "testing-unicloud" AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") @@ -98,6 +98,11 @@ def test_s3_download(self, test_file_content: str): self.s3.download(f"{MY_TEST_BUCKET}/{self.file_name}", download_path) # Verify the file content - assert download_path.read_text() == "Hello, world!" assert download_path.read_text() == test_file_content os.remove(download_path) + + def test_get_bucket(self): + """Test getting a bucket object.""" + bucket = self.s3.get_bucket(MY_TEST_BUCKET) + assert bucket.bucket.name == MY_TEST_BUCKET + assert isinstance(bucket, Bucket) From 01d43fe2ef41e052784d354b74ea5a5dc7672cce Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 00:02:04 +0100 Subject: [PATCH 05/33] reformat aws tests --- src/unicloud/aws/aws.py | 6 ++++-- tests/aws/test_aws.py | 27 ++++++--------------------- tests/conftest.py | 23 ++++++++++++++++++++++- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 6f68644..d271260 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -2,6 +2,8 @@ from typing import Optional import traceback +from pathlib import Path +from typing import Union import boto3 @@ -56,7 +58,7 @@ def create_client(self): aws_secret_access_key=self.aws_secret_access_key, ) - def upload(self, local_path: str, bucket_path: str): + def upload(self, local_path: Union[str, Path], bucket_path: str): """Upload a file to S3. Parameters @@ -75,7 +77,7 @@ def upload(self, local_path: str, bucket_path: str): raise e print(f"File {local_path} uploaded to {bucket_path}.") - def download(self, bucket_path: str, local_path: str): + def download(self, bucket_path: str, local_path: Union[str, Path]): """Download a file from S3. Parameters diff --git a/tests/aws/test_aws.py b/tests/aws/test_aws.py index aa3413b..5f0ba41 100644 --- a/tests/aws/test_aws.py +++ b/tests/aws/test_aws.py @@ -10,20 +10,6 @@ from unicloud.aws.aws import S3, Bucket MY_TEST_BUCKET = "testing-unicloud" -AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") -AWS_SECRET_ACCESS_KEY = os.getenv("aws_secret_access_key") -REGION = "eu-central-1" - - -@pytest.fixture -def boto_client() -> boto3.client: - return boto3.client( - "s3", - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=REGION, - ) - class TestS3: @@ -80,29 +66,28 @@ def test_download_data(self, test_file: str, test_file_content: str): class TestS3E2E: """End-to-end tests for the S3 class.""" - s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) file_name = "test_upload.txt" - def test_s3_upload(self, test_file: Path, boto_client: boto3.client): + def test_s3_upload(self, unicloud_s3, test_file: Path, boto_client: boto3.client): """Test file upload to S3.""" - self.s3.upload(test_file, f"{MY_TEST_BUCKET}/{self.file_name}") + unicloud_s3.upload(test_file, f"{MY_TEST_BUCKET}/{self.file_name}") # Verify the file exists in S3 response = boto_client.list_objects_v2(Bucket=MY_TEST_BUCKET) assert self.file_name in [obj["Key"] for obj in response["Contents"]] - def test_s3_download(self, test_file_content: str): + def test_s3_download(self, unicloud_s3, test_file_content: str): """Test file download from S3.""" download_path = Path("tests/data/aws-test-file.txt") - self.s3.download(f"{MY_TEST_BUCKET}/{self.file_name}", download_path) + unicloud_s3.download(f"{MY_TEST_BUCKET}/{self.file_name}", download_path) # Verify the file content assert download_path.read_text() == test_file_content os.remove(download_path) - def test_get_bucket(self): + def test_get_bucket(self, unicloud_s3): """Test getting a bucket object.""" - bucket = self.s3.get_bucket(MY_TEST_BUCKET) + bucket = unicloud_s3.get_bucket(MY_TEST_BUCKET) assert bucket.bucket.name == MY_TEST_BUCKET assert isinstance(bucket, Bucket) diff --git a/tests/conftest.py b/tests/conftest.py index a786429..af726f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,8 +2,20 @@ from pathlib import Path from typing import Dict +import boto3 import pytest +from unicloud.aws.aws import S3 + +AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") +AWS_SECRET_ACCESS_KEY = os.getenv("aws_secret_access_key") +REGION = "eu-central-1" + + +@pytest.fixture +def unicloud_s3() -> S3: + return S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) + @pytest.fixture def test_file() -> Path: @@ -15,7 +27,6 @@ def test_file_content() -> str: return "This is a test file.\n" - @pytest.fixture def upload_test_data() -> Dict[str, Path]: local_dir = Path("tests/data/upload-dir") @@ -30,3 +41,13 @@ def upload_test_data() -> Dict[str, Path]: "bucket_path": bucket_path, "expected_files": expected_files, } + + +@pytest.fixture +def boto_client() -> boto3.client: + return boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=REGION, + ) \ No newline at end of file From aa644b8c59a37e2832509432a4186b929efd0d8a Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 19:50:06 +0100 Subject: [PATCH 06/33] add `aws.Bucket.delete` and tests --- src/unicloud/aws/aws.py | 52 +++++++++++++++++++++++++++++++++++++++- tests/aws/test_bucket.py | 49 +++++++++++++++++++++++++++++++++++++ tests/conftest.py | 43 ++++++++++++++++++++++++++++++++- 3 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 tests/aws/test_bucket.py diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index d271260..6f568f7 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -1,9 +1,10 @@ """S3 Cloud Storage.""" from typing import Optional +import os import traceback from pathlib import Path -from typing import Union +from typing import List, Optional, Union import boto3 @@ -114,3 +115,52 @@ def __init__(self, bucket): # :boto3.resources("s3").Bucket def bucket(self): """bucket.""" return self._bucket + + def list_files(self, prefix: Optional[str] = None) -> List[str]: + """List files in the bucket.""" + if prefix is None: + prefix = "" + + return [obj.key for obj in self.bucket.objects.filter(Prefix=prefix)] + def delete(self, bucket_path: str): + """ + Delete a file or directory from the S3 bucket. + + Parameters + ---------- + bucket_path : str + Path in the bucket to delete. + + Raises + ------ + ValueError + If the file or directory does not exist. + + Notes + ----- + - Deletes a single file or all files within a directory. + + Examples + -------- + Delete a single file: + >>> bucket.delete('bucket/file.txt') + + Delete a directory: + >>> bucket.delete('bucket/dir/') + """ + if bucket_path.endswith("/"): + self._delete_directory(bucket_path) + else: + self._delete_file(bucket_path) + + def _delete_file(self, bucket_path: str): + """Delete a single file.""" + obj = self.bucket.Object(bucket_path) + obj.delete() + print(f"Deleted {bucket_path}.") + + def _delete_directory(self, bucket_path: str): + """Delete a directory recursively.""" + for obj in self.bucket.objects.filter(Prefix=bucket_path): + obj.delete() + print(f"Deleted {obj.key}.") \ No newline at end of file diff --git a/tests/aws/test_bucket.py b/tests/aws/test_bucket.py new file mode 100644 index 0000000..7c69d46 --- /dev/null +++ b/tests/aws/test_bucket.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import Dict + +import boto3 +import pytest + +from unicloud.aws.aws import Bucket + + +class TestDeleteE2E: + """ + End-to-End tests for the Bucket class delete method. + """ + + @pytest.fixture(autouse=True) + def setup(self, s3_bucket_name, aws_access_key_id, aws_secret_access_key, region): + """ + Setup a mock S3 bucket and temporary directory for testing. + """ + s3 = boto3.resource( + "s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region, + ) + self.bucket = Bucket(s3.Bucket(s3_bucket_name)) + + def test_delete_file(self, test_file): + """ + Test deleting a single file from the bucket. + """ + file_name = "test-delete-file.txt" + self.bucket.upload(test_file, file_name) + self.bucket.delete(file_name) + objects = [obj.key for obj in self.bucket.bucket.objects.all()] + assert file_name not in objects + + def test_delete_directory(self, upload_test_data: Dict[str, Path]): + """ + Test deleting a directory from the bucket. + """ + local_dir = upload_test_data["local_dir"] + bucket_path = upload_test_data["bucket_path"] + + self.bucket.upload(local_dir, f"{bucket_path}/") + self.bucket.delete(f"{bucket_path}/") + + objects = self.bucket.list_files(f"{bucket_path}/") + assert not objects diff --git a/tests/conftest.py b/tests/conftest.py index af726f1..b05aaf2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,27 @@ AWS_ACCESS_KEY_ID = os.getenv("aws_access_key_id") AWS_SECRET_ACCESS_KEY = os.getenv("aws_secret_access_key") REGION = "eu-central-1" +S3_BUCKET_NAME = "testing-unicloud" + + +@pytest.fixture +def aws_access_key_id() -> str: + return AWS_ACCESS_KEY_ID + + +@pytest.fixture +def aws_secret_access_key() -> str: + return AWS_SECRET_ACCESS_KEY + + +@pytest.fixture +def region() -> str: + return REGION + + +@pytest.fixture +def s3_bucket_name() -> str: + return S3_BUCKET_NAME @pytest.fixture @@ -50,4 +71,24 @@ def boto_client() -> boto3.client: aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region_name=REGION, - ) \ No newline at end of file + ) + + +@pytest.fixture +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + return { + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing", + "region_name": "us-east-1", + } + + +# @pytest.fixture +# def s3_client_mock(aws_credentials): +# """Create an S3 client for testing.""" +# with mock_aws(): +# boto3.client("s3", region_name=aws_credentials["region_name"]).create_bucket( +# Bucket="my-test-bucket" +# ) +# yield S3(**aws_credentials) From e0d4a3473eeac3c46a8af89f5bdedb9abf1c46bf Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 19:58:13 +0100 Subject: [PATCH 07/33] rename test_gcs_bucket.py to test_bucket.py --- tests/google_cloud/{test_gcs_bucket.py => test_bucket.py} | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename tests/google_cloud/{test_gcs_bucket.py => test_bucket.py} (98%) diff --git a/tests/google_cloud/test_gcs_bucket.py b/tests/google_cloud/test_bucket.py similarity index 98% rename from tests/google_cloud/test_gcs_bucket.py rename to tests/google_cloud/test_bucket.py index 5c9e6a7..92ca34a 100644 --- a/tests/google_cloud/test_gcs_bucket.py +++ b/tests/google_cloud/test_bucket.py @@ -38,6 +38,9 @@ def test_file_exists(self): assert not self.bucket.file_exists("non_existent_file.geojson") def test_upload_file(self, test_file: Path): + """ + Test uploading a single file to the bucket. + """ bucket_path = f"test-upload-gcs-bucket-{test_file.name}" self.bucket.upload(test_file, bucket_path) assert any(blob.name == bucket_path for blob in self.bucket.bucket.list_blobs()) @@ -196,7 +199,9 @@ def test_list_files_with_all_filters(self): class TestDeleteE2E: - + """ + End-to-End tests for the Bucket class delete method. + """ @pytest.fixture def gcs_bucket(self) -> Bucket: return GCS(PROJECT_ID).get_bucket(MY_TEST_BUCKET) From 9e6d161baa1fa6faa3e502b30a2f2c8167495104 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 20:19:26 +0100 Subject: [PATCH 08/33] add `aws.Bucket.upload` and tests --- src/unicloud/aws/aws.py | 63 ++++++++++++++++++++++++++++++++++++++++ tests/aws/test_bucket.py | 41 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 6f568f7..488bc47 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -122,6 +122,69 @@ def list_files(self, prefix: Optional[str] = None) -> List[str]: prefix = "" return [obj.key for obj in self.bucket.objects.filter(Prefix=prefix)] + def upload( + self, local_path: Union[str, Path], bucket_path: str, overwrite: bool = False + ): + """ + Upload a file or directory to the S3 bucket. + + Parameters + ---------- + local_path : Union[str, Path] + Path to the local file or directory to upload. + bucket_path : str + Path in the bucket to upload to. + overwrite : bool, optional + Whether to overwrite existing files. Default is False. + + Raises + ------ + FileNotFoundError + If the local path does not exist. + ValueError + If attempting to overwrite an existing file and overwrite is False. + + Notes + ----- + - Uploads a single file or all files within a directory (including subdirectories). + + Examples + -------- + Upload a single file: + >>> bucket.upload('local/file.txt', 'bucket/file.txt') + + Upload a directory: + >>> bucket.upload('local/dir', 'bucket/dir') + """ + local_path = Path(local_path) + if not local_path.exists(): + raise FileNotFoundError(f"Path {local_path} does not exist.") + + if local_path.is_file(): + self._upload_file(local_path, bucket_path, overwrite) + elif local_path.is_dir(): + self._upload_directory(local_path, bucket_path, overwrite) + else: + raise ValueError( + f"Invalid path type: {local_path} is neither a file nor a directory." + ) + + def _upload_file(self, local_path: Path, bucket_path: str, overwrite: bool): + """Upload a single file.""" + if not overwrite and self.file_exists(bucket_path): + raise ValueError(f"File {bucket_path} already exists in the bucket.") + self.bucket.upload_file(Filename=str(local_path), Key=bucket_path) + print(f"File {local_path} uploaded to {bucket_path}.") + + def _upload_directory(self, local_path: Path, bucket_path: str, overwrite: bool): + """Upload a directory recursively.""" + for root, _, files in os.walk(local_path): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(local_path) + s3_path = f"{bucket_path.rstrip('/')}/{relative_path.as_posix()}" + self._upload_file(file_path, s3_path, overwrite) + def delete(self, bucket_path: str): """ Delete a file or directory from the S3 bucket. diff --git a/tests/aws/test_bucket.py b/tests/aws/test_bucket.py index 7c69d46..a492c3f 100644 --- a/tests/aws/test_bucket.py +++ b/tests/aws/test_bucket.py @@ -7,6 +7,47 @@ from unicloud.aws.aws import Bucket +class TestBucketE2E: + """ + End-to-End tests for the Bucket class. + """ + + @pytest.fixture(autouse=True) + def setup(self, s3_bucket_name, aws_access_key_id, aws_secret_access_key, region): + """ + Setup a mock S3 bucket and temporary directory for testing. + """ + s3 = boto3.resource( + "s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region, + ) + self.bucket = Bucket(s3.Bucket(s3_bucket_name)) + + def test_upload_file(self, test_file: Path): + """ + Test uploading a single file to the bucket. + """ + file_name = "test-upload-file.txt" + self.bucket.upload(test_file, file_name) + objects = [obj.key for obj in self.bucket.bucket.objects.all()] + assert file_name in objects + self.bucket.delete(file_name) + + def test_upload_directory(self, upload_test_data: Dict[str, Path]): + """ + Test uploading a directory to the bucket. + """ + local_dir = upload_test_data["local_dir"] + bucket_path = upload_test_data["bucket_path"] + + self.bucket.upload(local_dir, f"{bucket_path}/") + objects = [obj.key for obj in self.bucket.bucket.objects.all()] + expected_files = upload_test_data["expected_files"] + assert set(objects) & expected_files == expected_files + self.bucket.delete(f"{bucket_path}/") + class TestDeleteE2E: """ End-to-End tests for the Bucket class delete method. From f5ff156c36243fe5b47548325bbe1c4cff3f9c42 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 22:13:21 +0100 Subject: [PATCH 09/33] give unique names to test files --- .../{test_bucket.py => test_aws_bucket.py} | 42 +++++++++++++++++++ .../{test_bucket.py => test_gcs_bucket.py} | 0 2 files changed, 42 insertions(+) rename tests/aws/{test_bucket.py => test_aws_bucket.py} (65%) rename tests/google_cloud/{test_bucket.py => test_gcs_bucket.py} (100%) diff --git a/tests/aws/test_bucket.py b/tests/aws/test_aws_bucket.py similarity index 65% rename from tests/aws/test_bucket.py rename to tests/aws/test_aws_bucket.py index a492c3f..6155550 100644 --- a/tests/aws/test_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from typing import Dict @@ -48,6 +49,47 @@ def test_upload_directory(self, upload_test_data: Dict[str, Path]): assert set(objects) & expected_files == expected_files self.bucket.delete(f"{bucket_path}/") + def test_download_file(self, test_file: Path, test_file_content: str): + """ + Test downloading a single file from the bucket. + """ + file_name = "test-download-file.txt" + self.bucket.upload(test_file, file_name, overwrite=True) + download_path = Path("tests/data/aws-downloaded-file.txt") + self.bucket.download(file_name, str(download_path)) + assert download_path.exists() + assert download_path.read_text() == test_file_content + self.bucket.delete(file_name) + download_path.unlink() + + def test_download_directory(self, upload_test_data: Dict[str, Path]): + """ + Test downloading a directory from the bucket. + """ + local_dir = upload_test_data["local_dir"] + bucket_path = "test-download-dir" + expected_files = upload_test_data["expected_files"] + + self.bucket.upload(local_dir, f"{bucket_path}/", overwrite=True) + + download_path = Path("tests/data/aws-downloaded-dir") + self.bucket.download(f"{bucket_path}/", str(download_path)) + + expected_files = [ + file.replace("upload-dir", download_path.name) for file in expected_files + ] + assert download_path.exists() + assert download_path.is_dir() + + actual_files = [ + str(file.relative_to(download_path.parent)).replace("\\", "/") + for file in download_path.rglob("*") + if file.is_file() + ] + assert set(actual_files) == set(expected_files) + shutil.rmtree(download_path) + + class TestDeleteE2E: """ End-to-End tests for the Bucket class delete method. diff --git a/tests/google_cloud/test_bucket.py b/tests/google_cloud/test_gcs_bucket.py similarity index 100% rename from tests/google_cloud/test_bucket.py rename to tests/google_cloud/test_gcs_bucket.py From dfd758eebd2b75d229aa85c6946d938915e7eed5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 22:23:15 +0100 Subject: [PATCH 10/33] add `aws.Bucket.download` and tests --- src/unicloud/aws/aws.py | 84 +++++++++++++++++++++++++-- tests/google_cloud/test_gcs_bucket.py | 1 + 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 488bc47..0da5c90 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -1,6 +1,5 @@ """S3 Cloud Storage.""" -from typing import Optional import os import traceback from pathlib import Path @@ -122,6 +121,7 @@ def list_files(self, prefix: Optional[str] = None) -> List[str]: prefix = "" return [obj.key for obj in self.bucket.objects.filter(Prefix=prefix)] + def upload( self, local_path: Union[str, Path], bucket_path: str, overwrite: bool = False ): @@ -151,10 +151,10 @@ def upload( Examples -------- Upload a single file: - >>> bucket.upload('local/file.txt', 'bucket/file.txt') + >>> bucket.upload('local/file.txt', 'bucket/file.txt') # doctest: +SKIP Upload a directory: - >>> bucket.upload('local/dir', 'bucket/dir') + >>> bucket.upload('local/dir', 'bucket/dir') # doctest: +SKIP """ local_path = Path(local_path) if not local_path.exists(): @@ -185,6 +185,61 @@ def _upload_directory(self, local_path: Path, bucket_path: str, overwrite: bool) s3_path = f"{bucket_path.rstrip('/')}/{relative_path.as_posix()}" self._upload_file(file_path, s3_path, overwrite) + def download( + self, bucket_path: str, local_path: Union[str, Path], overwrite: bool = False + ): + """ + Download a file or directory from the S3 bucket. + + Parameters + ---------- + bucket_path : str + Path in the bucket to download. + local_path : Union[str, Path] + Local path to save the downloaded file or directory. + overwrite : bool, optional + Whether to overwrite existing local files. Default is False. + + Raises + ------ + ValueError + If the local path exists and overwrite is False. + + Notes + ----- + - If bucket_path is a directory, downloads all files within it recursively. + + Examples + -------- + Download a single file: + >>> bucket.download('bucket/file.txt', 'local/file.txt') # doctest: +SKIP + + Download a directory: + >>> bucket.download('bucket/dir/', 'local/dir/') # doctest: +SKIP + """ + local_path = Path(local_path) + if bucket_path.endswith("/"): + self._download_directory(bucket_path, local_path, overwrite) + else: + self._download_file(bucket_path, local_path, overwrite) + + def _download_file(self, bucket_path: str, local_path: Path, overwrite: bool): + """Download a single file.""" + if local_path.exists() and not overwrite: + raise ValueError(f"File {local_path} already exists locally.") + local_path.parent.mkdir(parents=True, exist_ok=True) + self.bucket.download_file(Key=bucket_path, Filename=str(local_path)) + print(f"File {bucket_path} downloaded to {local_path}.") + + def _download_directory(self, bucket_path: str, local_path: Path, overwrite: bool): + """Download a directory recursively.""" + local_path.mkdir(parents=True, exist_ok=True) + for obj in self.bucket.objects.filter(Prefix=bucket_path): + if obj.key.endswith("/"): + continue + relative_path = Path(obj.key).relative_to(bucket_path) + self._download_file(obj.key, local_path / relative_path, overwrite) + def delete(self, bucket_path: str): """ Delete a file or directory from the S3 bucket. @@ -206,10 +261,10 @@ def delete(self, bucket_path: str): Examples -------- Delete a single file: - >>> bucket.delete('bucket/file.txt') + >>> bucket.delete('bucket/file.txt') # doctest: +SKIP Delete a directory: - >>> bucket.delete('bucket/dir/') + >>> bucket.delete('bucket/dir/') # doctest: +SKIP """ if bucket_path.endswith("/"): self._delete_directory(bucket_path) @@ -226,4 +281,21 @@ def _delete_directory(self, bucket_path: str): """Delete a directory recursively.""" for obj in self.bucket.objects.filter(Prefix=bucket_path): obj.delete() - print(f"Deleted {obj.key}.") \ No newline at end of file + print(f"Deleted {obj.key}.") + + def file_exists(self, bucket_path: str) -> bool: + """ + Check if a file exists in the bucket. + + Parameters + ---------- + bucket_path : str + Path in the bucket to check. + + Returns + ------- + bool + True if the file exists, False otherwise. + """ + objs = list(self.bucket.objects.filter(Prefix=bucket_path)) + return len(objs) > 0 and objs[0].key == bucket_path diff --git a/tests/google_cloud/test_gcs_bucket.py b/tests/google_cloud/test_gcs_bucket.py index 92ca34a..6fbccf4 100644 --- a/tests/google_cloud/test_gcs_bucket.py +++ b/tests/google_cloud/test_gcs_bucket.py @@ -202,6 +202,7 @@ class TestDeleteE2E: """ End-to-End tests for the Bucket class delete method. """ + @pytest.fixture def gcs_bucket(self) -> Bucket: return GCS(PROJECT_ID).get_bucket(MY_TEST_BUCKET) From f779f2ebace4f72ee11619c81a44b49ea94196fa Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 22:24:13 +0100 Subject: [PATCH 11/33] add docstring to the `aws.Bucket` class --- src/unicloud/aws/aws.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 0da5c90..4745194 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -104,10 +104,28 @@ def get_bucket(self, bucket_name: str) -> "Bucket": class Bucket: - """S3 Bucket.""" + """ + AWS S3 Bucket interface for file and directory operations. + + This class allows interacting with an S3 bucket for uploading, downloading, + and deleting files and directories. + """ def __init__(self, bucket): # :boto3.resources("s3").Bucket - """Initialize the S3 bucket.""" + """ + Initialize the Bucket class. + + Parameters + ---------- + bucket : boto3.resources.factory.s3.Bucket + A boto3 S3 Bucket resource instance. + + Examples + -------- + >>> import boto3 + >>> s3 = boto3.resource("s3") + >>> bucket = Bucket(s3.Bucket("my-bucket")) + """ self._bucket = bucket @property @@ -115,6 +133,11 @@ def bucket(self): """bucket.""" return self._bucket + @property + def name(self): + """Bucket name.""" + return self.bucket.name + def list_files(self, prefix: Optional[str] = None) -> List[str]: """List files in the bucket.""" if prefix is None: From a3b2d8ef720ecc79329bb04de17ce385eb14e73d Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 22:25:10 +0100 Subject: [PATCH 12/33] add `google_cloud.gcs.Bucket.name` property --- src/unicloud/google_cloud/gcs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/unicloud/google_cloud/gcs.py b/src/unicloud/google_cloud/gcs.py index 37ae8d3..2d5d9af 100644 --- a/src/unicloud/google_cloud/gcs.py +++ b/src/unicloud/google_cloud/gcs.py @@ -269,11 +269,16 @@ def __init__(self, bucket: storage.bucket.Bucket): def __str__(self): """__str__.""" - return f"Bucket: {self.bucket.name}" + return f"Bucket: {self.name}" def __repr__(self): """__repr__.""" - return f"Bucket: {self.bucket.name}" + return f"Bucket: {self.name}" + + @property + def name(self): + """name.""" + return self.bucket.name @property def bucket(self) -> storage.bucket.Bucket: From 73a70b073ade4a000c447378dcec14f2c5c65b19 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 22:45:41 +0100 Subject: [PATCH 13/33] add mock test for the aws.Bucket class --- src/unicloud/aws/aws.py | 2 + tests/aws/test_aws_bucket.py | 101 +++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 4745194..992a301 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -250,7 +250,9 @@ def _download_file(self, bucket_path: str, local_path: Path, overwrite: bool): """Download a single file.""" if local_path.exists() and not overwrite: raise ValueError(f"File {local_path} already exists locally.") + local_path.parent.mkdir(parents=True, exist_ok=True) + self.bucket.download_file(Key=bucket_path, Filename=str(local_path)) print(f"File {bucket_path} downloaded to {local_path}.") diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 6155550..1e595af 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -1,6 +1,7 @@ import shutil from pathlib import Path from typing import Dict +from unittest.mock import MagicMock, patch import boto3 import pytest @@ -130,3 +131,103 @@ def test_delete_directory(self, upload_test_data: Dict[str, Path]): objects = self.bucket.list_files(f"{bucket_path}/") assert not objects + + +class TestBucketMock: + """ + Mock tests for the Bucket class. + """ + + def setup_method(self): + """ + Setup a mocked S3 Bucket instance. + """ + self.mock_bucket = MagicMock() + self.bucket = Bucket(self.mock_bucket) + + def test_upload_file(self): + """ + Test uploading a single file to the bucket using mocks. + """ + local_file = Path("test.txt") + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.is_file", return_value=True + ): + self.bucket.upload(local_file, "test.txt") + self.mock_bucket.upload_file.assert_called_once_with( + Filename=str(local_file), Key="test.txt" + ) + + def test_upload_directory(self): + """ + Test uploading a directory to the bucket using mocks. + """ + local_dir = Path("test_dir") + files = [local_dir / "file1.txt", local_dir / "file2.txt"] + + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.is_dir", return_value=True + ), patch( + "os.walk", return_value=[(str(local_dir), [], [f.name for f in files])] + ): + self.bucket.upload(local_dir, "test_dir/") + + for file in files: + s3_path = f"test_dir/{file.name}" + self.mock_bucket.upload_file.assert_any_call( + Filename=str(file), Key=s3_path + ) + + def test_download_file(self): + """ + Test downloading a single file from the bucket using mocks. + """ + local_file = Path("downloaded.txt") + with patch("pathlib.Path.exists", return_value=False), patch( + "pathlib.Path.mkdir" + ): + self.bucket.download("test.txt", str(local_file)) + + self.mock_bucket.download_file.assert_called_once_with( + Key="test.txt", Filename=str(local_file) + ) + + def test_download_directory(self): + """ + Test downloading a directory from the bucket using mocks. + """ + local_dir = Path("downloaded_dir") + mock_objects = [ + MagicMock(key="test_dir/file1.txt"), + MagicMock(key="test_dir/file2.txt"), + ] + self.mock_bucket.objects.filter.return_value = mock_objects + + with patch("pathlib.Path.mkdir") as mock_mkdir: + self.bucket.download("test_dir/", str(local_dir)) + + for obj in mock_objects: + expected_path = local_dir / Path(obj.key).relative_to("test_dir/") + self.mock_bucket.download_file.assert_any_call( + Key=obj.key, Filename=str(expected_path) + ) + + def test_delete_file(self): + """ + Test deleting a single file from the bucket using mocks. + """ + self.bucket.delete("test.txt") + self.mock_bucket.Object.return_value.delete.assert_called_once() + + def test_delete_directory(self): + """ + Test deleting a directory from the bucket using mocks. + """ + mock_objects = [ + MagicMock(key="test_dir/file1.txt"), + MagicMock(key="test_dir/file2.txt"), + ] + self.mock_bucket.objects.filter.return_value = mock_objects + self.bucket.delete("test_dir/") + for obj in mock_objects: + obj.delete.assert_called_once() From 6bdba85b32339595baa4607dab1aaa4bba9ede8a Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 23:05:50 +0100 Subject: [PATCH 14/33] add test upload existing file --- tests/aws/test_aws_bucket.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 1e595af..1bf08bb 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -50,6 +50,23 @@ def test_upload_directory(self, upload_test_data: Dict[str, Path]): assert set(objects) & expected_files == expected_files self.bucket.delete(f"{bucket_path}/") + def test_upload_overwrite(self, test_file: Path): + """ + Test uploading a file with overwrite behavior. + """ + file_name = "test-upload-overwrite.txt" + self.bucket.upload(test_file, file_name) + + # Overwrite = False + with pytest.raises(ValueError, match="File .* already exists."): + self.bucket.upload(test_file, file_name, overwrite=False) + + # Overwrite = True + self.bucket.upload(test_file, file_name, overwrite=True) + objects = [obj.key for obj in self.bucket.bucket.objects.all()] + assert file_name in objects + self.bucket.delete(file_name) + def test_download_file(self, test_file: Path, test_file_content: str): """ Test downloading a single file from the bucket. From d7e07b8f7e6420cb453b0070a0e50db38463fdda Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 22 Dec 2024 23:31:52 +0100 Subject: [PATCH 15/33] the delete method raise an error if the given directory is empty --- src/unicloud/aws/aws.py | 6 +++++- tests/aws/test_aws_bucket.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 992a301..125718f 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -304,7 +304,11 @@ def _delete_file(self, bucket_path: str): def _delete_directory(self, bucket_path: str): """Delete a directory recursively.""" - for obj in self.bucket.objects.filter(Prefix=bucket_path): + objects = list(self.bucket.objects.filter(Prefix=bucket_path)) + if not objects: + raise ValueError(f"No files found in the directory: {bucket_path}") + + for obj in objects: obj.delete() print(f"Deleted {obj.key}.") diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 1bf08bb..0f90bb1 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -149,6 +149,16 @@ def test_delete_directory(self, upload_test_data: Dict[str, Path]): objects = self.bucket.list_files(f"{bucket_path}/") assert not objects + def test_delete_empty_directory(self): + """ + Test attempting to delete an empty directory in the bucket. + """ + empty_dir = "empty-dir/" + with pytest.raises( + ValueError, match=f"No files found in the directory: {empty_dir}" + ): + self.bucket.delete(empty_dir) + class TestBucketMock: """ @@ -248,3 +258,13 @@ def test_delete_directory(self): self.bucket.delete("test_dir/") for obj in mock_objects: obj.delete.assert_called_once() + + def test_delete_empty_directory_mock(self): + """ + Test deleting an empty directory using mocks. + """ + self.mock_bucket.objects.filter.return_value = [] + with pytest.raises( + ValueError, match="No files found in the directory: empty-dir/" + ): + self.bucket.delete("empty-dir/") From 6ace5b35feb29d8d92e04467b7460eac5e591505 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:01:35 +0100 Subject: [PATCH 16/33] the delete method raises an error if the given file does not exist --- src/unicloud/aws/aws.py | 8 +++++--- tests/aws/test_aws_bucket.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 125718f..4a459e0 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -298,9 +298,11 @@ def delete(self, bucket_path: str): def _delete_file(self, bucket_path: str): """Delete a single file.""" - obj = self.bucket.Object(bucket_path) - obj.delete() - print(f"Deleted {bucket_path}.") + objects = list(self.bucket.objects.filter(Prefix=bucket_path)) + if not objects or objects[0].key != bucket_path: + raise ValueError(f"File {bucket_path} not found in the bucket.") + self.bucket.Object(bucket_path).delete() + print(f"Deleted: {bucket_path}") def _delete_directory(self, bucket_path: str): """Delete a directory recursively.""" diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 0f90bb1..cee82e1 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -159,6 +159,16 @@ def test_delete_empty_directory(self): ): self.bucket.delete(empty_dir) + def test_delete_nonexistent_file(self): + """ + Test attempting to delete a nonexistent file in the bucket. + """ + nonexistent_file = "nonexistent-file.txt" + with pytest.raises( + ValueError, match=f"File {nonexistent_file} not found in the bucket." + ): + self.bucket.delete(nonexistent_file) + class TestBucketMock: """ @@ -243,6 +253,9 @@ def test_delete_file(self): """ Test deleting a single file from the bucket using mocks. """ + object_mock = MagicMock() + object_mock.key = "test.txt" + self.mock_bucket.objects.filter.return_value = [object_mock] self.bucket.delete("test.txt") self.mock_bucket.Object.return_value.delete.assert_called_once() @@ -268,3 +281,18 @@ def test_delete_empty_directory_mock(self): ValueError, match="No files found in the directory: empty-dir/" ): self.bucket.delete("empty-dir/") + + def test_delete_nonexistent_file_mock(self): + """ + Test deleting a nonexistent file using mocks. + """ + self.mock_bucket.objects.filter.return_value = [] + + with pytest.raises( + ValueError, match="File nonexistent-file.txt not found in the bucket." + ): + self.bucket.delete("nonexistent-file.txt") + + self.mock_bucket.objects.filter.assert_called_once_with( + Prefix="nonexistent-file.txt" + ) From 793b7fc562f5b6ae6122b5f5d9ff3be05ab6b3a5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:06:22 +0100 Subject: [PATCH 17/33] reformat delete tests --- tests/aws/test_aws_bucket.py | 134 +++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index cee82e1..48d70b2 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -108,68 +108,6 @@ def test_download_directory(self, upload_test_data: Dict[str, Path]): shutil.rmtree(download_path) -class TestDeleteE2E: - """ - End-to-End tests for the Bucket class delete method. - """ - - @pytest.fixture(autouse=True) - def setup(self, s3_bucket_name, aws_access_key_id, aws_secret_access_key, region): - """ - Setup a mock S3 bucket and temporary directory for testing. - """ - s3 = boto3.resource( - "s3", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - region_name=region, - ) - self.bucket = Bucket(s3.Bucket(s3_bucket_name)) - - def test_delete_file(self, test_file): - """ - Test deleting a single file from the bucket. - """ - file_name = "test-delete-file.txt" - self.bucket.upload(test_file, file_name) - self.bucket.delete(file_name) - objects = [obj.key for obj in self.bucket.bucket.objects.all()] - assert file_name not in objects - - def test_delete_directory(self, upload_test_data: Dict[str, Path]): - """ - Test deleting a directory from the bucket. - """ - local_dir = upload_test_data["local_dir"] - bucket_path = upload_test_data["bucket_path"] - - self.bucket.upload(local_dir, f"{bucket_path}/") - self.bucket.delete(f"{bucket_path}/") - - objects = self.bucket.list_files(f"{bucket_path}/") - assert not objects - - def test_delete_empty_directory(self): - """ - Test attempting to delete an empty directory in the bucket. - """ - empty_dir = "empty-dir/" - with pytest.raises( - ValueError, match=f"No files found in the directory: {empty_dir}" - ): - self.bucket.delete(empty_dir) - - def test_delete_nonexistent_file(self): - """ - Test attempting to delete a nonexistent file in the bucket. - """ - nonexistent_file = "nonexistent-file.txt" - with pytest.raises( - ValueError, match=f"File {nonexistent_file} not found in the bucket." - ): - self.bucket.delete(nonexistent_file) - - class TestBucketMock: """ Mock tests for the Bucket class. @@ -249,6 +187,78 @@ def test_download_directory(self): Key=obj.key, Filename=str(expected_path) ) + +class TestDeleteE2E: + """ + End-to-End tests for the Bucket class delete method. + """ + + @pytest.fixture(autouse=True) + def setup(self, s3_bucket_name, aws_access_key_id, aws_secret_access_key, region): + """ + Setup a mock S3 bucket and temporary directory for testing. + """ + s3 = boto3.resource( + "s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region, + ) + self.bucket = Bucket(s3.Bucket(s3_bucket_name)) + + def test_delete_file(self, test_file): + """ + Test deleting a single file from the bucket. + """ + file_name = "test-delete-file.txt" + self.bucket.upload(test_file, file_name) + self.bucket.delete(file_name) + objects = [obj.key for obj in self.bucket.bucket.objects.all()] + assert file_name not in objects + + def test_delete_directory(self, upload_test_data: Dict[str, Path]): + """ + Test deleting a directory from the bucket. + """ + local_dir = upload_test_data["local_dir"] + bucket_path = upload_test_data["bucket_path"] + + self.bucket.upload(local_dir, f"{bucket_path}/") + self.bucket.delete(f"{bucket_path}/") + + objects = self.bucket.list_files(f"{bucket_path}/") + assert not objects + + def test_delete_empty_directory(self): + """ + Test attempting to delete an empty directory in the bucket. + """ + empty_dir = "empty-dir/" + with pytest.raises( + ValueError, match=f"No files found in the directory: {empty_dir}" + ): + self.bucket.delete(empty_dir) + + def test_delete_nonexistent_file(self): + """ + Test attempting to delete a nonexistent file in the bucket. + """ + nonexistent_file = "nonexistent-file.txt" + with pytest.raises( + ValueError, match=f"File {nonexistent_file} not found in the bucket." + ): + self.bucket.delete(nonexistent_file) + + +class TestDeleteMock: + + def setup_method(self): + """ + Setup a mocked S3 Bucket instance. + """ + self.mock_bucket = MagicMock() + self.bucket = Bucket(self.mock_bucket) + def test_delete_file(self): """ Test deleting a single file from the bucket using mocks. From 3ec1333a16c70178b9dd2d77af25b181f88e615b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:06:59 +0100 Subject: [PATCH 18/33] add docstring --- src/unicloud/aws/aws.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 4a459e0..16e0b9c 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -139,7 +139,19 @@ def name(self): return self.bucket.name def list_files(self, prefix: Optional[str] = None) -> List[str]: - """List files in the bucket.""" + """ + List files in the bucket with a specific prefix. + + Parameters + ---------- + prefix : str, optional, default is None + The prefix to filter the files (default is "", which lists all files). + + Returns + ------- + list of str + List of file keys matching the prefix. + """ if prefix is None: prefix = "" From de8b7d40a6b271bcdb5966f24c50f77401111a00 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:09:28 +0100 Subject: [PATCH 19/33] restucture test classes --- tests/aws/test_aws_bucket.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 48d70b2..9e73db3 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -108,7 +108,7 @@ def test_download_directory(self, upload_test_data: Dict[str, Path]): shutil.rmtree(download_path) -class TestBucketMock: +class TestUploadMock: """ Mock tests for the Bucket class. """ @@ -153,6 +153,15 @@ def test_upload_directory(self): Filename=str(file), Key=s3_path ) + +class TestDownloadMock: + def setup_method(self): + """ + Setup a mocked S3 Bucket instance. + """ + self.mock_bucket = MagicMock() + self.bucket = Bucket(self.mock_bucket) + def test_download_file(self): """ Test downloading a single file from the bucket using mocks. From beccd3859188ebf8e85c051a878134e8ad7a6ad0 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:25:18 +0100 Subject: [PATCH 20/33] test the cases of the overwrite parameter in the `download` method --- tests/aws/test_aws_bucket.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 9e73db3..730e192 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -107,6 +107,25 @@ def test_download_directory(self, upload_test_data: Dict[str, Path]): assert set(actual_files) == set(expected_files) shutil.rmtree(download_path) + def test_download_overwrite(self, test_file: Path): + """ + Test downloading a file with overwrite behavior. + """ + file_name = "test-download-overwrite.txt" + download_path = Path("tests/data/aws-downloaded-file.txt") + + self.bucket.upload(test_file, file_name) + self.bucket.download(file_name, str(download_path)) + + # Overwrite = False + with pytest.raises(ValueError, match="File .* already exists locally."): + self.bucket.download(file_name, str(download_path), overwrite=False) + + # Overwrite = True + self.bucket.download(file_name, str(download_path), overwrite=True) + self.bucket.delete(file_name) + download_path.unlink() + class TestUploadMock: """ From 0c675ba73d000035235872585d21ac6ceaada461 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:40:21 +0100 Subject: [PATCH 21/33] the `upload` method raises an error if the directory is empty --- src/unicloud/aws/aws.py | 3 +++ tests/aws/test_aws_bucket.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 16e0b9c..e1da9ba 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -213,6 +213,9 @@ def _upload_file(self, local_path: Path, bucket_path: str, overwrite: bool): def _upload_directory(self, local_path: Path, bucket_path: str, overwrite: bool): """Upload a directory recursively.""" + if local_path.is_dir() and not any(local_path.iterdir()): + raise ValueError(f"Directory {local_path} is empty.") + for root, _, files in os.walk(local_path): for file in files: file_path = Path(root) / file diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 730e192..f40f995 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -67,6 +67,16 @@ def test_upload_overwrite(self, test_file: Path): assert file_name in objects self.bucket.delete(file_name) + def test_upload_empty_directory(self): + """ + Test uploading an empty directory to the bucket. + """ + empty_dir = Path("tests/data/empty-dir") + empty_dir.mkdir(parents=True, exist_ok=True) + with pytest.raises(ValueError, match="Directory .* is empty."): + self.bucket.upload(empty_dir, "empty-dir/") + shutil.rmtree(empty_dir) + def test_download_file(self, test_file: Path, test_file_content: str): """ Test downloading a single file from the bucket. @@ -161,7 +171,7 @@ def test_upload_directory(self): with patch("pathlib.Path.exists", return_value=True), patch( "pathlib.Path.is_dir", return_value=True - ), patch( + ), patch("pathlib.Path.iterdir", return_value=files), patch( "os.walk", return_value=[(str(local_dir), [], [f.name for f in files])] ): self.bucket.upload(local_dir, "test_dir/") From e0fda5f9317456f4d5b7961fe990281d77fef286 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:45:57 +0100 Subject: [PATCH 22/33] add docstring to the upload method --- src/unicloud/aws/aws.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index e1da9ba..dc57e97 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -178,6 +178,9 @@ def upload( If the local path does not exist. ValueError If attempting to overwrite an existing file and overwrite is False. + ValueError + If the local path is a directory and it is empty. + Notes ----- From a3a557a55c5d8194f1b7122b4faad0f1ea8e0a9c Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 00:55:56 +0100 Subject: [PATCH 23/33] the `download` method will raise an error if the download directory is empty --- src/unicloud/aws/aws.py | 3 +++ tests/aws/test_aws_bucket.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index dc57e97..4a084b1 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -276,6 +276,9 @@ def _download_file(self, bucket_path: str, local_path: Path, overwrite: bool): def _download_directory(self, bucket_path: str, local_path: Path, overwrite: bool): """Download a directory recursively.""" + if not any(self.list_files(bucket_path)): + raise ValueError(f"Directory {bucket_path} is empty.") + local_path.mkdir(parents=True, exist_ok=True) for obj in self.bucket.objects.filter(Prefix=bucket_path): if obj.key.endswith("/"): diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index f40f995..5c0c6a4 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -136,6 +136,13 @@ def test_download_overwrite(self, test_file: Path): self.bucket.delete(file_name) download_path.unlink() + def test_download_empty_directory(self): + """ + Test downloading an empty directory from the bucket. + """ + with pytest.raises(ValueError, match="Directory .* is empty."): + self.bucket.download("empty-dir/", "tests/data/empty-dir/") + class TestUploadMock: """ From 47a7aef5ce5b4d7baf1b46b724eba87aad0afafe Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 01:07:17 +0100 Subject: [PATCH 24/33] add mock tests for the `upload` and `download` methods for empty directories --- src/unicloud/aws/aws.py | 2 ++ tests/aws/test_aws_bucket.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 4a084b1..504f537 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -245,6 +245,8 @@ def download( ------ ValueError If the local path exists and overwrite is False. + ValueError + If the file or directory does not exist in the bucket. Notes ----- diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 5c0c6a4..3e5798b 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -189,6 +189,17 @@ def test_upload_directory(self): Filename=str(file), Key=s3_path ) + def test_upload_empty_directory_mock(self): + """ + Test uploading an empty directory to the bucket using mocks. + """ + empty_dir = Path("test-empty-dir") + with patch("pathlib.Path.exists", return_value=True), patch( + "pathlib.Path.is_dir", return_value=True + ), patch("pathlib.Path.iterdir", return_value=[]): + with pytest.raises(ValueError, match="Directory .* is empty."): + self.bucket.upload(empty_dir, "empty-dir/") + class TestDownloadMock: def setup_method(self): @@ -232,6 +243,16 @@ def test_download_directory(self): Key=obj.key, Filename=str(expected_path) ) + def test_download_empty_directory_mock(self): + """ + Test downloading an empty directory using mocks. + """ + with patch("pathlib.Path.mkdir"), patch( + "unicloud.aws.aws.Bucket.list_files", return_value=[] + ): + with pytest.raises(ValueError, match="Directory .* is empty."): + self.bucket.download("empty-dir/", "local-empty-dir/") + class TestDeleteE2E: """ From ad888fea6aae7097cf1fd7c1224179244085f3d7 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 01:16:52 +0100 Subject: [PATCH 25/33] the `upload` method raises an error if the directory is empty --- src/unicloud/google_cloud/gcs.py | 5 +++++ tests/google_cloud/test_gcs_bucket.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/unicloud/google_cloud/gcs.py b/src/unicloud/google_cloud/gcs.py index 2d5d9af..d21a6b7 100644 --- a/src/unicloud/google_cloud/gcs.py +++ b/src/unicloud/google_cloud/gcs.py @@ -533,6 +533,8 @@ def _upload_directory( If the `local_path` does not exist or is not a directory. ValueError If any destination file exists in the bucket and `overwrite` is False. + ValueError + If the directory is empty. Examples -------- @@ -558,6 +560,9 @@ def _upload_directory( ... print(e) "The file 'bucket/folder/subdir/file.txt' already exists in the bucket and overwrite is set to False." """ + if local_path.is_dir() and not any(local_path.iterdir()): + raise ValueError(f"Directory {local_path} is empty.") + for file in local_path.rglob("*"): if file.is_file(): relative_path = file.relative_to(local_path) diff --git a/tests/google_cloud/test_gcs_bucket.py b/tests/google_cloud/test_gcs_bucket.py index 6fbccf4..cab3b4a 100644 --- a/tests/google_cloud/test_gcs_bucket.py +++ b/tests/google_cloud/test_gcs_bucket.py @@ -345,9 +345,9 @@ def test_upload_directory_with_subdirectories(self): with ( patch("pathlib.Path.exists", return_value=True), patch("pathlib.Path.is_dir", return_value=True), + patch("pathlib.Path.iterdir", return_value=files), patch("pathlib.Path.rglob", return_value=files), ): - # Mock individual file checks and uploads with patch("pathlib.Path.is_file", side_effect=[False, True, True, True]): gcs_bucket.upload(local_directory, bucket_path, overwrite=True) From bd60569db23113d2c21620c0c3be8d7c35884ae5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 23 Dec 2024 01:27:42 +0100 Subject: [PATCH 26/33] correct the test marker issues in pyproject.toml --- poetry.lock | 327 +++++++++++++++++++++++-------------------------- pyproject.toml | 15 +-- 2 files changed, 161 insertions(+), 181 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7b9fd90..e6b4b7f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -42,32 +42,32 @@ files = [ [[package]] name = "attrs" -version = "24.2.0" +version = "24.3.0" description = "Classes Without Boilerplate" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, - {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, ] [package.extras] benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.3.2" +version = "1.4.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "Authlib-1.3.2-py2.py3-none-any.whl", hash = "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"}, - {file = "authlib-1.3.2.tar.gz", hash = "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2"}, + {file = "Authlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"}, + {file = "authlib-1.4.0.tar.gz", hash = "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2"}, ] [package.dependencies] @@ -178,17 +178,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.35.81" +version = "1.35.86" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.81-py3-none-any.whl", hash = "sha256:742941b2424c0223d2d94a08c3485462fa7c58d816b62ca80f08e555243acee1"}, - {file = "boto3-1.35.81.tar.gz", hash = "sha256:d2e95fa06f095b8e0c545dd678c6269d253809b2997c30f5ce8a956c410b4e86"}, + {file = "boto3-1.35.86-py3-none-any.whl", hash = "sha256:ed59fb4883da167464a5dfbc96e76d571db75e1a7a27d8e7b790c3008b02fcc7"}, + {file = "boto3-1.35.86.tar.gz", hash = "sha256:d61476fdd5a5388503b72c897083310d2329ce088593c4332b571a860be5d155"}, ] [package.dependencies] -botocore = ">=1.35.81,<1.36.0" +botocore = ">=1.35.86,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -197,13 +197,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.81" +version = "1.35.86" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.81-py3-none-any.whl", hash = "sha256:a7b13bbd959bf2d6f38f681676aab408be01974c46802ab997617b51399239f7"}, - {file = "botocore-1.35.81.tar.gz", hash = "sha256:564c2478e50179e0b766e6a87e5e0cdd35e1bc37eb375c1cf15511f5dd13600d"}, + {file = "botocore-1.35.86-py3-none-any.whl", hash = "sha256:77cb4b445e4f424f956c68c688bd3ad527f4d214d51d67ffc8e245f4476d7de0"}, + {file = "botocore-1.35.86.tar.gz", hash = "sha256:951e944eb30284b4593d4da98f70f7b5292ea237e4de0c5a2852946a549b8347"}, ] [package.dependencies] @@ -453,13 +453,13 @@ files = [ [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -656,19 +656,19 @@ poetry = ["poetry"] [[package]] name = "filelock" -version = "3.12.4" +version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.12.4-py3-none-any.whl", hash = "sha256:08c21d87ded6e2b9da6728c3dff51baf1dcecf973b768ef35bcbc3447edb9ad4"}, - {file = "filelock-3.12.4.tar.gz", hash = "sha256:2e6f249f1f3654291606e046b09f1fd5eac39b360664c27f5aad072012f8bcbd"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"] -typing = ["typing-extensions (>=4.7.1)"] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] +typing = ["typing-extensions (>=4.12.2)"] [[package]] name = "flake8" @@ -798,13 +798,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.155.0" +version = "2.156.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.155.0-py2.py3-none-any.whl", hash = "sha256:83fe9b5aa4160899079d7c93a37be306546a17e6686e2549bcc9584f1a229747"}, - {file = "google_api_python_client-2.155.0.tar.gz", hash = "sha256:25529f89f0d13abcf3c05c089c423fb2858ac16e0b3727543393468d0d7af67c"}, + {file = "google_api_python_client-2.156.0-py2.py3-none-any.whl", hash = "sha256:6352185c505e1f311f11b0b96c1b636dcb0fec82cd04b80ac5a671ac4dcab339"}, + {file = "google_api_python_client-2.156.0.tar.gz", hash = "sha256:b809c111ded61716a9c1c7936e6899053f13bae3defcdfda904bd2ca68065b9c"}, ] [package.dependencies] @@ -1034,13 +1034,13 @@ files = [ [[package]] name = "jinja2" -version = "3.1.4" +version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, ] [package.dependencies] @@ -1174,13 +1174,13 @@ files = [ [[package]] name = "marshmallow" -version = "3.23.1" +version = "3.23.2" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" files = [ - {file = "marshmallow-3.23.1-py3-none-any.whl", hash = "sha256:fece2eb2c941180ea1b7fcbd4a83c51bfdd50093fdd3ad2585ee5e1df2508491"}, - {file = "marshmallow-3.23.1.tar.gz", hash = "sha256:3a8dfda6edd8dcdbf216c0ede1d1e78d230a6dc9c5a088f58c4083b974a0d468"}, + {file = "marshmallow-3.23.2-py3-none-any.whl", hash = "sha256:bcaf2d6fd74fb1459f8450e85d994997ad3e70036452cbfa4ab685acb19479b3"}, + {file = "marshmallow-3.23.2.tar.gz", hash = "sha256:c448ac6455ca4d794773f00bae22c2f351d62d739929f761dce5eacb5c468d7f"}, ] [package.dependencies] @@ -1215,19 +1215,19 @@ files = [ [[package]] name = "moto" -version = "5.0.22" +version = "5.0.24" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "moto-5.0.22-py3-none-any.whl", hash = "sha256:defae32e834ba5674f77cbbe996b41dc248dd81289af8032fa3e847284409b29"}, - {file = "moto-5.0.22.tar.gz", hash = "sha256:daf47b8a1f5f190cd3eaa40018a643f38e542277900cf1db7f252cedbfed998f"}, + {file = "moto-5.0.24-py3-none-any.whl", hash = "sha256:4d826f1574849f18ddd2fcbf614d97f82c8fddfb9d95fac1078da01a39b57c10"}, + {file = "moto-5.0.24.tar.gz", hash = "sha256:dba6426bd770fbb9d892633fbd35253cbc181eeaa0eba97d6f058720a8fe9b42"}, ] [package.dependencies] boto3 = ">=1.9.201" botocore = ">=1.14.0,<1.35.45 || >1.35.45,<1.35.46 || >1.35.46" -cryptography = ">=3.3.1" +cryptography = ">=35.0.0" Jinja2 = ">=2.10.1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" @@ -1261,48 +1261,48 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] name = "mypy" -version = "1.13.0" +version = "1.14.0" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, - {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, - {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, - {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, - {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, - {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, - {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, - {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, - {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, - {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, - {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, - {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, - {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, - {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, - {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, - {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, - {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, - {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, - {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, - {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, - {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, - {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, + {file = "mypy-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e971c1c667007f9f2b397ffa80fa8e1e0adccff336e5e77e74cb5f22868bee87"}, + {file = "mypy-1.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e86aaeaa3221a278c66d3d673b297232947d873773d61ca3ee0e28b2ff027179"}, + {file = "mypy-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1628c5c3ce823d296e41e2984ff88c5861499041cb416a8809615d0c1f41740e"}, + {file = "mypy-1.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7fadb29b77fc14a0dd81304ed73c828c3e5cde0016c7e668a86a3e0dfc9f3af3"}, + {file = "mypy-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:3fa76988dc760da377c1e5069200a50d9eaaccf34f4ea18428a3337034ab5a44"}, + {file = "mypy-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e73c8a154eed31db3445fe28f63ad2d97b674b911c00191416cf7f6459fd49a"}, + {file = "mypy-1.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:273e70fcb2e38c5405a188425aa60b984ffdcef65d6c746ea5813024b68c73dc"}, + {file = "mypy-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1daca283d732943731a6a9f20fdbcaa927f160bc51602b1d4ef880a6fb252015"}, + {file = "mypy-1.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7e68047bedb04c1c25bba9901ea46ff60d5eaac2d71b1f2161f33107e2b368eb"}, + {file = "mypy-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:7a52f26b9c9b1664a60d87675f3bae00b5c7f2806e0c2800545a32c325920bcc"}, + {file = "mypy-1.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d5326ab70a6db8e856d59ad4cb72741124950cbbf32e7b70e30166ba7bbf61dd"}, + {file = "mypy-1.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bf4ec4980bec1e0e24e5075f449d014011527ae0055884c7e3abc6a99cd2c7f1"}, + {file = "mypy-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:390dfb898239c25289495500f12fa73aa7f24a4c6d90ccdc165762462b998d63"}, + {file = "mypy-1.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7e026d55ddcd76e29e87865c08cbe2d0104e2b3153a523c529de584759379d3d"}, + {file = "mypy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:585ed36031d0b3ee362e5107ef449a8b5dfd4e9c90ccbe36414ee405ee6b32ba"}, + {file = "mypy-1.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9f6f4c0b27401d14c483c622bc5105eff3911634d576bbdf6695b9a7c1ba741"}, + {file = "mypy-1.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b2280cedcb312c7a79f5001ae5325582d0d339bce684e4a529069d0e7ca1e7"}, + {file = "mypy-1.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:342de51c48bab326bfc77ce056ba08c076d82ce4f5a86621f972ed39970f94d8"}, + {file = "mypy-1.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00df23b42e533e02a6f0055e54de9a6ed491cd8b7ea738647364fd3a39ea7efc"}, + {file = "mypy-1.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e8c8387e5d9dff80e7daf961df357c80e694e942d9755f3ad77d69b0957b8e3f"}, + {file = "mypy-1.14.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b16738b1d80ec4334654e89e798eb705ac0c36c8a5c4798496cd3623aa02286"}, + {file = "mypy-1.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:10065fcebb7c66df04b05fc799a854b1ae24d9963c8bb27e9064a9bdb43aa8ad"}, + {file = "mypy-1.14.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbb7d683fa6bdecaa106e8368aa973ecc0ddb79a9eaeb4b821591ecd07e9e03c"}, + {file = "mypy-1.14.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3498cb55448dc5533e438cd13d6ddd28654559c8c4d1fd4b5ca57a31b81bac01"}, + {file = "mypy-1.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:c7b243408ea43755f3a21a0a08e5c5ae30eddb4c58a80f415ca6b118816e60aa"}, + {file = "mypy-1.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:14117b9da3305b39860d0aa34b8f1ff74d209a368829a584eb77524389a9c13e"}, + {file = "mypy-1.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af98c5a958f9c37404bd4eef2f920b94874507e146ed6ee559f185b8809c44cc"}, + {file = "mypy-1.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0b343a1d3989547024377c2ba0dca9c74a2428ad6ed24283c213af8dbb0710b"}, + {file = "mypy-1.14.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cdb5563c1726c85fb201be383168f8c866032db95e1095600806625b3a648cb7"}, + {file = "mypy-1.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:74e925649c1ee0a79aa7448baf2668d81cc287dc5782cff6a04ee93f40fb8d3f"}, + {file = "mypy-1.14.0-py3-none-any.whl", hash = "sha256:2238d7f93fc4027ed1efc944507683df3ba406445a2b6c96e79666a045aadfab"}, + {file = "mypy-1.14.0.tar.gz", hash = "sha256:822dbd184d4a9804df5a7d5335a68cf7662930e70b8c1bc976645d1509f9a9d6"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" -typing-extensions = ">=4.6.0" +mypy_extensions = ">=1.0.0" +typing_extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -1335,66 +1335,48 @@ files = [ [[package]] name = "numpy" -version = "2.2.0" +version = "2.2.1" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" files = [ - {file = "numpy-2.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1e25507d85da11ff5066269d0bd25d06e0a0f2e908415534f3e603d2a78e4ffa"}, - {file = "numpy-2.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a62eb442011776e4036af5c8b1a00b706c5bc02dc15eb5344b0c750428c94219"}, - {file = "numpy-2.2.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:b606b1aaf802e6468c2608c65ff7ece53eae1a6874b3765f69b8ceb20c5fa78e"}, - {file = "numpy-2.2.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:36b2b43146f646642b425dd2027730f99bac962618ec2052932157e213a040e9"}, - {file = "numpy-2.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fe8f3583e0607ad4e43a954e35c1748b553bfe9fdac8635c02058023277d1b3"}, - {file = "numpy-2.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:122fd2fcfafdefc889c64ad99c228d5a1f9692c3a83f56c292618a59aa60ae83"}, - {file = "numpy-2.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3f2f5cddeaa4424a0a118924b988746db6ffa8565e5829b1841a8a3bd73eb59a"}, - {file = "numpy-2.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7fe4bb0695fe986a9e4deec3b6857003b4cfe5c5e4aac0b95f6a658c14635e31"}, - {file = "numpy-2.2.0-cp310-cp310-win32.whl", hash = "sha256:b30042fe92dbd79f1ba7f6898fada10bdaad1847c44f2dff9a16147e00a93661"}, - {file = "numpy-2.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:54dc1d6d66f8d37843ed281773c7174f03bf7ad826523f73435deb88ba60d2d4"}, - {file = "numpy-2.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9874bc2ff574c40ab7a5cbb7464bf9b045d617e36754a7bc93f933d52bd9ffc6"}, - {file = "numpy-2.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0da8495970f6b101ddd0c38ace92edea30e7e12b9a926b57f5fabb1ecc25bb90"}, - {file = "numpy-2.2.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:0557eebc699c1c34cccdd8c3778c9294e8196df27d713706895edc6f57d29608"}, - {file = "numpy-2.2.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:3579eaeb5e07f3ded59298ce22b65f877a86ba8e9fe701f5576c99bb17c283da"}, - {file = "numpy-2.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40deb10198bbaa531509aad0cd2f9fadb26c8b94070831e2208e7df543562b74"}, - {file = "numpy-2.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2aed8fcf8abc3020d6a9ccb31dbc9e7d7819c56a348cc88fd44be269b37427e"}, - {file = "numpy-2.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a222d764352c773aa5ebde02dd84dba3279c81c6db2e482d62a3fa54e5ece69b"}, - {file = "numpy-2.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4e58666988605e251d42c2818c7d3d8991555381be26399303053b58a5bbf30d"}, - {file = "numpy-2.2.0-cp311-cp311-win32.whl", hash = "sha256:4723a50e1523e1de4fccd1b9a6dcea750c2102461e9a02b2ac55ffeae09a4410"}, - {file = "numpy-2.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:16757cf28621e43e252c560d25b15f18a2f11da94fea344bf26c599b9cf54b73"}, - {file = "numpy-2.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cff210198bb4cae3f3c100444c5eaa573a823f05c253e7188e1362a5555235b3"}, - {file = "numpy-2.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b92a5828bd4d9aa0952492b7de803135038de47343b2aa3cc23f3b71a3dc4e"}, - {file = "numpy-2.2.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:ebe5e59545401fbb1b24da76f006ab19734ae71e703cdb4a8b347e84a0cece67"}, - {file = "numpy-2.2.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e2b8cd48a9942ed3f85b95ca4105c45758438c7ed28fff1e4ce3e57c3b589d8e"}, - {file = "numpy-2.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57fcc997ffc0bef234b8875a54d4058afa92b0b0c4223fc1f62f24b3b5e86038"}, - {file = "numpy-2.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ad7d11b309bd132d74397fcf2920933c9d1dc865487128f5c03d580f2c3d03"}, - {file = "numpy-2.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cb24cca1968b21355cc6f3da1a20cd1cebd8a023e3c5b09b432444617949085a"}, - {file = "numpy-2.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0798b138c291d792f8ea40fe3768610f3c7dd2574389e37c3f26573757c8f7ef"}, - {file = "numpy-2.2.0-cp312-cp312-win32.whl", hash = "sha256:afe8fb968743d40435c3827632fd36c5fbde633b0423da7692e426529b1759b1"}, - {file = "numpy-2.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:3a4199f519e57d517ebd48cb76b36c82da0360781c6a0353e64c0cac30ecaad3"}, - {file = "numpy-2.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f8c8b141ef9699ae777c6278b52c706b653bf15d135d302754f6b2e90eb30367"}, - {file = "numpy-2.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f0986e917aca18f7a567b812ef7ca9391288e2acb7a4308aa9d265bd724bdae"}, - {file = "numpy-2.2.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:1c92113619f7b272838b8d6702a7f8ebe5edea0df48166c47929611d0b4dea69"}, - {file = "numpy-2.2.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5a145e956b374e72ad1dff82779177d4a3c62bc8248f41b80cb5122e68f22d13"}, - {file = "numpy-2.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18142b497d70a34b01642b9feabb70156311b326fdddd875a9981f34a369b671"}, - {file = "numpy-2.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d41d1612c1a82b64697e894b75db6758d4f21c3ec069d841e60ebe54b5b571"}, - {file = "numpy-2.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a98f6f20465e7618c83252c02041517bd2f7ea29be5378f09667a8f654a5918d"}, - {file = "numpy-2.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e09d40edfdb4e260cb1567d8ae770ccf3b8b7e9f0d9b5c2a9992696b30ce2742"}, - {file = "numpy-2.2.0-cp313-cp313-win32.whl", hash = "sha256:3905a5fffcc23e597ee4d9fb3fcd209bd658c352657548db7316e810ca80458e"}, - {file = "numpy-2.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a184288538e6ad699cbe6b24859206e38ce5fba28f3bcfa51c90d0502c1582b2"}, - {file = "numpy-2.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7832f9e8eb00be32f15fdfb9a981d6955ea9adc8574c521d48710171b6c55e95"}, - {file = "numpy-2.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f0dd071b95bbca244f4cb7f70b77d2ff3aaaba7fa16dc41f58d14854a6204e6c"}, - {file = "numpy-2.2.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:b0b227dcff8cdc3efbce66d4e50891f04d0a387cce282fe1e66199146a6a8fca"}, - {file = "numpy-2.2.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ab153263a7c5ccaf6dfe7e53447b74f77789f28ecb278c3b5d49db7ece10d6d"}, - {file = "numpy-2.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e500aba968a48e9019e42c0c199b7ec0696a97fa69037bea163b55398e390529"}, - {file = "numpy-2.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:440cfb3db4c5029775803794f8638fbdbf71ec702caf32735f53b008e1eaece3"}, - {file = "numpy-2.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a55dc7a7f0b6198b07ec0cd445fbb98b05234e8b00c5ac4874a63372ba98d4ab"}, - {file = "numpy-2.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4bddbaa30d78c86329b26bd6aaaea06b1e47444da99eddac7bf1e2fab717bd72"}, - {file = "numpy-2.2.0-cp313-cp313t-win32.whl", hash = "sha256:30bf971c12e4365153afb31fc73f441d4da157153f3400b82db32d04de1e4066"}, - {file = "numpy-2.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:d35717333b39d1b6bb8433fa758a55f1081543de527171543a2b710551d40881"}, - {file = "numpy-2.2.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e12c6c1ce84628c52d6367863773f7c8c8241be554e8b79686e91a43f1733773"}, - {file = "numpy-2.2.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:b6207dc8fb3c8cb5668e885cef9ec7f70189bec4e276f0ff70d5aa078d32c88e"}, - {file = "numpy-2.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a50aeff71d0f97b6450d33940c7181b08be1441c6c193e678211bff11aa725e7"}, - {file = "numpy-2.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:df12a1f99b99f569a7c2ae59aa2d31724e8d835fc7f33e14f4792e3071d11221"}, - {file = "numpy-2.2.0.tar.gz", hash = "sha256:140dd80ff8981a583a60980be1a655068f8adebf7a45a06a6858c873fcdcd4a0"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675"}, + {file = "numpy-2.2.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957"}, + {file = "numpy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2"}, + {file = "numpy-2.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528"}, + {file = "numpy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95"}, + {file = "numpy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb"}, + {file = "numpy-2.2.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73"}, + {file = "numpy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8"}, + {file = "numpy-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0"}, + {file = "numpy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd"}, + {file = "numpy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315"}, + {file = "numpy-2.2.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7"}, + {file = "numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51"}, + {file = "numpy-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046"}, + {file = "numpy-2.2.1-cp312-cp312-win32.whl", hash = "sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2"}, + {file = "numpy-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e"}, + {file = "numpy-2.2.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59"}, + {file = "numpy-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af"}, + {file = "numpy-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51"}, ] [[package]] @@ -1619,52 +1601,53 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "5.29.1" +version = "5.29.2" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-5.29.1-cp310-abi3-win32.whl", hash = "sha256:22c1f539024241ee545cbcb00ee160ad1877975690b16656ff87dde107b5f110"}, - {file = "protobuf-5.29.1-cp310-abi3-win_amd64.whl", hash = "sha256:1fc55267f086dd4050d18ef839d7bd69300d0d08c2a53ca7df3920cc271a3c34"}, - {file = "protobuf-5.29.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d473655e29c0c4bbf8b69e9a8fb54645bc289dead6d753b952e7aa660254ae18"}, - {file = "protobuf-5.29.1-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5ba1d0e4c8a40ae0496d0e2ecfdbb82e1776928a205106d14ad6985a09ec155"}, - {file = "protobuf-5.29.1-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ee1461b3af56145aca2800e6a3e2f928108c749ba8feccc6f5dd0062c410c0d"}, - {file = "protobuf-5.29.1-cp38-cp38-win32.whl", hash = "sha256:50879eb0eb1246e3a5eabbbe566b44b10348939b7cc1b267567e8c3d07213853"}, - {file = "protobuf-5.29.1-cp38-cp38-win_amd64.whl", hash = "sha256:027fbcc48cea65a6b17028510fdd054147057fa78f4772eb547b9274e5219331"}, - {file = "protobuf-5.29.1-cp39-cp39-win32.whl", hash = "sha256:5a41deccfa5e745cef5c65a560c76ec0ed8e70908a67cc8f4da5fce588b50d57"}, - {file = "protobuf-5.29.1-cp39-cp39-win_amd64.whl", hash = "sha256:012ce28d862ff417fd629285aca5d9772807f15ceb1a0dbd15b88f58c776c98c"}, - {file = "protobuf-5.29.1-py3-none-any.whl", hash = "sha256:32600ddb9c2a53dedc25b8581ea0f1fd8ea04956373c0c07577ce58d312522e0"}, - {file = "protobuf-5.29.1.tar.gz", hash = "sha256:683be02ca21a6ffe80db6dd02c0b5b2892322c59ca57fd6c872d652cb80549cb"}, + {file = "protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851"}, + {file = "protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9"}, + {file = "protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb"}, + {file = "protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e"}, + {file = "protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e"}, + {file = "protobuf-5.29.2-cp38-cp38-win32.whl", hash = "sha256:e621a98c0201a7c8afe89d9646859859be97cb22b8bf1d8eacfd90d5bda2eb19"}, + {file = "protobuf-5.29.2-cp38-cp38-win_amd64.whl", hash = "sha256:13d6d617a2a9e0e82a88113d7191a1baa1e42c2cc6f5f1398d3b054c8e7e714a"}, + {file = "protobuf-5.29.2-cp39-cp39-win32.whl", hash = "sha256:36000f97ea1e76e8398a3f02936aac2a5d2b111aae9920ec1b769fc4a222c4d9"}, + {file = "protobuf-5.29.2-cp39-cp39-win_amd64.whl", hash = "sha256:2d2e674c58a06311c8e99e74be43e7f3a8d1e2b2fdf845eaa347fbd866f23355"}, + {file = "protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181"}, + {file = "protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e"}, ] [[package]] name = "psutil" -version = "6.0.0" +version = "6.1.1" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"}, - {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"}, - {file = "psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"}, - {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"}, - {file = "psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"}, - {file = "psutil-6.0.0-cp27-none-win32.whl", hash = "sha256:02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"}, - {file = "psutil-6.0.0-cp27-none-win_amd64.whl", hash = "sha256:21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"}, - {file = "psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"}, - {file = "psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"}, - {file = "psutil-6.0.0-cp36-cp36m-win32.whl", hash = "sha256:fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"}, - {file = "psutil-6.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"}, - {file = "psutil-6.0.0-cp37-abi3-win32.whl", hash = "sha256:a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"}, - {file = "psutil-6.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"}, - {file = "psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"}, - {file = "psutil-6.0.0.tar.gz", hash = "sha256:8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"}, + {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"}, + {file = "psutil-6.1.1-cp27-none-win32.whl", hash = "sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"}, + {file = "psutil-6.1.1-cp27-none-win_amd64.whl", hash = "sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"}, + {file = "psutil-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"}, + {file = "psutil-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"}, + {file = "psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"}, + {file = "psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"}, + {file = "psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"}, ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +test = ["pytest", "pytest-xdist", "setuptools"] [[package]] name = "pyasn1" @@ -2232,31 +2215,31 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "safety" -version = "3.2.13" +version = "3.2.14" description = "Checks installed dependencies for known vulnerabilities and licenses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "safety-3.2.13-py3-none-any.whl", hash = "sha256:c259d43b751c62973b484ed03b78409890236023a50d8e3292b80d84139a9d35"}, - {file = "safety-3.2.13.tar.gz", hash = "sha256:9328510c3286d67a788346d60df531b8bd2f35abbed3ac4dfaf8d119e6eec1ae"}, + {file = "safety-3.2.14-py3-none-any.whl", hash = "sha256:23ceeb06038ff65607c7f1311bffa3e92b029148b367b360ad8287d9f3395194"}, + {file = "safety-3.2.14.tar.gz", hash = "sha256:7a45d88b1903c5b7c370eaeb6ca131a52f147e0b8a0b302265f82824ef92adc7"}, ] [package.dependencies] Authlib = ">=1.2.0" Click = ">=8.0.2" dparse = ">=0.6.4" -filelock = ">=3.12.2,<3.13.0" +filelock = ">=3.16.1,<3.17.0" jinja2 = ">=3.1.0" marshmallow = ">=3.15.0" packaging = ">=21.0" -psutil = ">=6.0.0,<6.1.0" -pydantic = ">=1.10.12" +psutil = ">=6.1.0,<6.2.0" +pydantic = ">=2.6.0,<2.10.0" requests = "*" rich = "*" "ruamel.yaml" = ">=0.17.21" safety_schemas = "0.0.10" setuptools = ">=65.5.1" -typer = "*" +typer = ">=0.12.1" typing-extensions = ">=4.7.1" urllib3 = ">=1.26.5" @@ -2559,13 +2542,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.3" +version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] [package.extras] diff --git a/pyproject.toml b/pyproject.toml index 37ba2a1..fe0580a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,12 +77,6 @@ show_column_numbers = true show_error_codes = true show_error_context = true -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "-ra -q" -testpaths = [ - "tests", -] [tool.flake8] ignore = ["E501", "E203", "E266", "W503"] @@ -94,11 +88,14 @@ count = true max-complexity = 18 #select = B,C,E,F,W,T4 - +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra -q" +testpaths = [ + "tests", +] markers = [ - "fast: marks tests as fast (deselect with '-m \"not fast\"')", "e2e: marks tests as end-to-end (deselect with '-m \"e2e\"')", - "serial", "mock: marks tests as mock (deselect with '-m \"mock\"')", ] From 8e1354a5a5ea3d78ba12e3938f61136e62ab11bf Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:00:42 +0100 Subject: [PATCH 27/33] update docstring --- src/unicloud/aws/aws.py | 99 ++++++++++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 504f537..bab82dd 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -122,9 +122,18 @@ def __init__(self, bucket): # :boto3.resources("s3").Bucket Examples -------- - >>> import boto3 - >>> s3 = boto3.resource("s3") - >>> bucket = Bucket(s3.Bucket("my-bucket")) + - Initialize the Bucket class with a boto3 S3 Bucket resource instance: + + >>> import boto3 + >>> s3 = boto3.resource("s3") + >>> bucket = Bucket(s3.Bucket("my-bucket")) # doctest: +SKIP + + - Get the Bucket object from an S3 client: + >>> AWS_ACCESS_KEY_ID = "your-access key" + >>> AWS_SECRET_ACCESS_KEY = "your-secret-key" + >>> REGION = "us-east-1" + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP """ self._bucket = bucket @@ -144,13 +153,26 @@ def list_files(self, prefix: Optional[str] = None) -> List[str]: Parameters ---------- - prefix : str, optional, default is None - The prefix to filter the files (default is "", which lists all files). + prefix : str, optional, default=None + The prefix to filter files (e.g., 'folder/' to list files under 'folder/'). Returns ------- - list of str - List of file keys matching the prefix. + List[str] + A list of file keys matching the prefix. + + + Examples + -------- + Create the S3 client and bucket: + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP + + List all files in the bucket: + >>> bucket.list_files() # doctest: +SKIP + + List files with a specific prefix: + >>> bucket.list_files(prefix="folder/") # doctest: +SKIP """ if prefix is None: prefix = "" @@ -168,31 +190,33 @@ def upload( local_path : Union[str, Path] Path to the local file or directory to upload. bucket_path : str - Path in the bucket to upload to. - overwrite : bool, optional - Whether to overwrite existing files. Default is False. + The destination path in the bucket. + overwrite : bool, optional, default=False + Whether to overwrite existing files in the bucket. Raises ------ FileNotFoundError - If the local path does not exist. - ValueError - If attempting to overwrite an existing file and overwrite is False. + If the local file or directory does not exist. ValueError + If attempting to overwrite an existing file when `overwrite` is False. If the local path is a directory and it is empty. - Notes ----- - - Uploads a single file or all files within a directory (including subdirectories). + - Uploads a single file or recursively uploads a directory and its contents. Examples -------- + Create the S3 client and bucket: + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP + Upload a single file: - >>> bucket.upload('local/file.txt', 'bucket/file.txt') # doctest: +SKIP + >>> bucket.upload("local/file.txt", "bucket/file.txt", overwrite=False) # doctest: +SKIP Upload a directory: - >>> bucket.upload('local/dir', 'bucket/dir') # doctest: +SKIP + >>> bucket.upload("local/dir", "bucket/dir", overwrite=True) # doctest: +SKIP """ local_path = Path(local_path) if not local_path.exists(): @@ -237,28 +261,31 @@ def download( bucket_path : str Path in the bucket to download. local_path : Union[str, Path] - Local path to save the downloaded file or directory. - overwrite : bool, optional - Whether to overwrite existing local files. Default is False. + Local destination path for the downloaded file or directory. + overwrite : bool, optional, default=False + Whether to overwrite existing local files. Raises ------ ValueError - If the local path exists and overwrite is False. - ValueError + If the local path exists and `overwrite` is False. If the file or directory does not exist in the bucket. Notes ----- - - If bucket_path is a directory, downloads all files within it recursively. + - Downloads a single file or recursively downloads all files in a directory. Examples -------- + Create the S3 client and bucket: + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP + Download a single file: - >>> bucket.download('bucket/file.txt', 'local/file.txt') # doctest: +SKIP + >>> bucket.download("bucket/file.txt", "local/file.txt", overwrite=False) # doctest: +SKIP Download a directory: - >>> bucket.download('bucket/dir/', 'local/dir/') # doctest: +SKIP + >>> bucket.download("bucket/dir/", "local/dir/", overwrite=True) # doctest: +SKIP """ local_path = Path(local_path) if bucket_path.endswith("/"): @@ -295,24 +322,29 @@ def delete(self, bucket_path: str): Parameters ---------- bucket_path : str - Path in the bucket to delete. + The file or directory path in the bucket to delete. + - If it ends with '/', it is treated as a directory. Raises ------ ValueError - If the file or directory does not exist. + If the file or directory does not exist in the bucket. Notes ----- - - Deletes a single file or all files within a directory. + - Deletes a single file or recursively deletes all files in a directory. Examples -------- + Create the S3 client and bucket: + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP + Delete a single file: - >>> bucket.delete('bucket/file.txt') # doctest: +SKIP + >>> bucket.delete("bucket/file.txt") # doctest: +SKIP Delete a directory: - >>> bucket.delete('bucket/dir/') # doctest: +SKIP + >>> bucket.delete("bucket/dir/") # doctest: +SKIP """ if bucket_path.endswith("/"): self._delete_directory(bucket_path) @@ -344,12 +376,17 @@ def file_exists(self, bucket_path: str) -> bool: Parameters ---------- bucket_path : str - Path in the bucket to check. + The path of the file in the bucket. Returns ------- bool True if the file exists, False otherwise. + + Examples + -------- + Check if a file exists in the bucket: + >>> bucket.file_exists("bucket/file.txt") # doctest: +SKIP """ objs = list(self.bucket.objects.filter(Prefix=bucket_path)) return len(objs) > 0 and objs[0].key == bucket_path From f30f0ac64d567cc567b88844ac31680639b67717 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:09:43 +0100 Subject: [PATCH 28/33] create `AbstractBucket` class and use it as a superclass for the `aws.Bucket` --- src/unicloud/abstract_class.py | 35 ++++++++++++++++++++++++++++++++++ src/unicloud/aws/aws.py | 4 ++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/unicloud/abstract_class.py b/src/unicloud/abstract_class.py index 6db0820..267bac0 100644 --- a/src/unicloud/abstract_class.py +++ b/src/unicloud/abstract_class.py @@ -30,3 +30,38 @@ def download(self, source, file_path): - file_path: The path to save the downloaded file. """ pass + + +class AbstractBucket(ABC): + """Abstract class for cloud storage bucket.""" + + @abstractmethod + def upload(self): + """Upload a file/directory to the bucket.""" + pass + + @abstractmethod + def download(self): + """Download a file/directory from the bucket.""" + pass + + @abstractmethod + def delete(self): + """Delete a file/directory from the bucket.""" + pass + + @abstractmethod + def list_files(self): + """List the files/directory in the bucket.""" + pass + + @abstractmethod + def file_exists(self): + """Check if a file/directory exists in the bucket.""" + pass + + @property + @abstractmethod + def name(self): + """Get the name of the bucket.""" + pass diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index bab82dd..8c8b906 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -7,7 +7,7 @@ import boto3 -from unicloud.abstract_class import CloudStorageFactory +from unicloud.abstract_class import AbstractBucket, CloudStorageFactory class S3(CloudStorageFactory): @@ -103,7 +103,7 @@ def get_bucket(self, bucket_name: str) -> "Bucket": return Bucket(bucket) -class Bucket: +class Bucket(AbstractBucket): """ AWS S3 Bucket interface for file and directory operations. From 18aca0c52e170a3c19f374c53f6f881bc96e3ada Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:17:42 +0100 Subject: [PATCH 29/33] add parameters to the `AbstractBucket` class and unify it for the subclasses --- src/unicloud/abstract_class.py | 15 ++++++++++++--- src/unicloud/google_cloud/gcs.py | 30 +++++++++++++++--------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/unicloud/abstract_class.py b/src/unicloud/abstract_class.py index 267bac0..e70d9bf 100644 --- a/src/unicloud/abstract_class.py +++ b/src/unicloud/abstract_class.py @@ -1,6 +1,8 @@ """This module contains the abstract class for cloud storage factory.""" from abc import ABC, abstractmethod +from pathlib import Path +from typing import Union class CloudStorageFactory(ABC): @@ -36,17 +38,24 @@ class AbstractBucket(ABC): """Abstract class for cloud storage bucket.""" @abstractmethod - def upload(self): + def upload( + self, + local_path: Union[str, Path], + bucket_path: Union[str, Path], + overwrite: bool = False, + ): """Upload a file/directory to the bucket.""" pass @abstractmethod - def download(self): + def download( + self, bucket_path: str, local_path: Union[str, Path], overwrite: bool = False + ): """Download a file/directory from the bucket.""" pass @abstractmethod - def delete(self): + def delete(self, bucket_path: str): """Delete a file/directory from the bucket.""" pass diff --git a/src/unicloud/google_cloud/gcs.py b/src/unicloud/google_cloud/gcs.py index d21a6b7..bc22eb1 100644 --- a/src/unicloud/google_cloud/gcs.py +++ b/src/unicloud/google_cloud/gcs.py @@ -8,7 +8,7 @@ from google.cloud import storage from google.oauth2 import service_account -from unicloud.abstract_class import CloudStorageFactory +from unicloud.abstract_class import AbstractBucket, CloudStorageFactory from unicloud.utils import decode @@ -260,7 +260,7 @@ def get_bucket(self, bucket_id: str) -> "Bucket": return Bucket(bucket) -class Bucket: +class Bucket(AbstractBucket): """GCSBucket.""" def __init__(self, bucket: storage.bucket.Bucket): @@ -571,7 +571,7 @@ def _upload_directory( ) self._upload_file(file, bucket_file_path, overwrite) - def download(self, file_name: str, local_path: str, overwrite: bool = False): + def download(self, bucket_path: str, local_path: str, overwrite: bool = False): """Download a file from GCS. Downloads a file from a Google Cloud Storage bucket to a local directory or path. @@ -582,7 +582,7 @@ def download(self, file_name: str, local_path: str, overwrite: bool = False): Parameters ---------- - file_name : str + bucket_path : str The name of the file or directory to download from the GCS bucket. - For a single file, provide its name (e.g., "example.txt"). - For a directory, provide its name ending with a '/' (e.g., "data/"). @@ -631,10 +631,10 @@ def download(self, file_name: str, local_path: str, overwrite: bool = False): upload : To upload a file from a local path to a GCS bucket. """ - if file_name.endswith("/"): - self._download_directory(file_name, local_path, overwrite) + if bucket_path.endswith("/"): + self._download_directory(bucket_path, local_path, overwrite) else: - self._download_file(file_name, local_path, overwrite) + self._download_file(bucket_path, local_path, overwrite) def _download_file( self, bucket_path: str, local_path: Union[str, Path], overwrite: bool = False @@ -745,7 +745,7 @@ def _download_directory( blob.download_to_filename(local_file_path) print(f"File '{blob.name}' downloaded to '{local_file_path}'.") - def delete(self, file_path: str): + def delete(self, bucket_path: str): """ Delete a file or all files in a directory from the GCS bucket. @@ -755,7 +755,7 @@ def delete(self, file_path: str): Parameters ---------- - file_path : str + bucket_path : str The path to the file or directory in the GCS bucket. - For a single file, provide the file name (e.g., "example.txt"). - For a directory, provide the path ending with '/' (e.g., "data/"). @@ -784,9 +784,9 @@ def delete(self, file_path: str): - For directories, all files and subdirectories are deleted recursively. - Deleting a non-existent file or directory raises a `ValueError`. """ - if file_path.endswith("/"): + if bucket_path.endswith("/"): # Delete all files in the directory - blobs = self.bucket.list_blobs(prefix=file_path) + blobs = self.bucket.list_blobs(prefix=bucket_path) deleted_files = [] for blob in blobs: blob.delete() @@ -794,12 +794,12 @@ def delete(self, file_path: str): print(f"Deleted file: {blob.name}") if not deleted_files: - raise ValueError(f"No files found in the directory: {file_path}") + raise ValueError(f"No files found in the directory: {bucket_path}") else: # Delete a single file - blob = self.bucket.blob(file_path) + blob = self.bucket.blob(bucket_path) if blob.exists(): blob.delete() - print(f"Blob {file_path} deleted.") + print(f"Blob {bucket_path} deleted.") else: - raise ValueError(f"File {file_path} not found in the bucket.") + raise ValueError(f"File {bucket_path} not found in the bucket.") From 6a14d750a6f62bcaaadae1ba531e6205c2ebcaf5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:21:51 +0100 Subject: [PATCH 30/33] refactor the `delete` method --- src/unicloud/google_cloud/gcs.py | 38 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/unicloud/google_cloud/gcs.py b/src/unicloud/google_cloud/gcs.py index bc22eb1..e81c8a5 100644 --- a/src/unicloud/google_cloud/gcs.py +++ b/src/unicloud/google_cloud/gcs.py @@ -785,21 +785,25 @@ def delete(self, bucket_path: str): - Deleting a non-existent file or directory raises a `ValueError`. """ if bucket_path.endswith("/"): - # Delete all files in the directory - blobs = self.bucket.list_blobs(prefix=bucket_path) - deleted_files = [] - for blob in blobs: - blob.delete() - deleted_files.append(blob.name) - print(f"Deleted file: {blob.name}") - - if not deleted_files: - raise ValueError(f"No files found in the directory: {bucket_path}") + self._delete_directory(bucket_path) else: - # Delete a single file - blob = self.bucket.blob(bucket_path) - if blob.exists(): - blob.delete() - print(f"Blob {bucket_path} deleted.") - else: - raise ValueError(f"File {bucket_path} not found in the bucket.") + self._delete_file(bucket_path) + + def _delete_directory(self, bucket_path: str): + blobs = self.bucket.list_blobs(prefix=bucket_path) + deleted_files = [] + for blob in blobs: + blob.delete() + deleted_files.append(blob.name) + print(f"Deleted file: {blob.name}") + + if not deleted_files: + raise ValueError(f"No files found in the directory: {bucket_path}") + + def _delete_file(self, bucket_path: str): + blob = self.bucket.blob(bucket_path) + if blob.exists(): + blob.delete() + print(f"Blob {bucket_path} deleted.") + else: + raise ValueError(f"File {bucket_path} not found in the bucket.") From 1664d7b410ec1ceb29ff097177a8f8a819e870c6 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:33:27 +0100 Subject: [PATCH 31/33] add `__str__` and `__repr__` to both gcs and aws --- src/unicloud/aws/aws.py | 8 ++++++++ tests/aws/test_aws_bucket.py | 17 +++++++++++++++++ tests/google_cloud/test_gcs_bucket.py | 17 +++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index 8c8b906..a45412c 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -137,6 +137,14 @@ def __init__(self, bucket): # :boto3.resources("s3").Bucket """ self._bucket = bucket + def __str__(self): + """__str__.""" + return f"Bucket: {self.name}" + + def __repr__(self): + """__repr__.""" + return f"Bucket: {self.name}" + @property def bucket(self): """bucket.""" diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 3e5798b..333d90f 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -144,6 +144,23 @@ def test_download_empty_directory(self): self.bucket.download("empty-dir/", "tests/data/empty-dir/") +class TestPropertiesMock: + + def setup_method(self): + self.mock_bucket = MagicMock() + self.mock_bucket.name = "test_bucket" + self.gcs_bucket = Bucket(self.mock_bucket) + + def test_name_property(self): + assert self.gcs_bucket.name == "test_bucket" + + def test__str__(self): + assert str(self.gcs_bucket) == "Bucket: test_bucket" + + def test__repr__(self): + assert str(self.gcs_bucket.__repr__()) == "Bucket: test_bucket" + + class TestUploadMock: """ Mock tests for the Bucket class. diff --git a/tests/google_cloud/test_gcs_bucket.py b/tests/google_cloud/test_gcs_bucket.py index cab3b4a..6f01c9a 100644 --- a/tests/google_cloud/test_gcs_bucket.py +++ b/tests/google_cloud/test_gcs_bucket.py @@ -249,6 +249,23 @@ def test_delete_empty_directory_e2e(self, gcs_bucket: Bucket): gcs_bucket.delete(directory) +class TestPropertiesMock: + + def setup_method(self): + self.mock_bucket = MagicMock() + self.mock_bucket.name = "test_bucket" + self.gcs_bucket = Bucket(self.mock_bucket) + + def test_name_property(self): + assert self.gcs_bucket.name == "test_bucket" + + def test__str__(self): + assert str(self.gcs_bucket) == "Bucket: test_bucket" + + def test__repr__(self): + assert str(self.gcs_bucket.__repr__()) == "Bucket: test_bucket" + + class TestDownloadMock: @pytest.mark.mock From fdcb32a060a402b69090c8c011407c1f6ae2daa4 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:36:34 +0100 Subject: [PATCH 32/33] add `__str__` and `__repr__` to the abstract bucket class --- src/unicloud/abstract_class.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/unicloud/abstract_class.py b/src/unicloud/abstract_class.py index e70d9bf..3c91699 100644 --- a/src/unicloud/abstract_class.py +++ b/src/unicloud/abstract_class.py @@ -37,6 +37,16 @@ def download(self, source, file_path): class AbstractBucket(ABC): """Abstract class for cloud storage bucket.""" + @abstractmethod + def __str__(self): + """Return the name of the bucket.""" + pass + + @abstractmethod + def __repr__(self): + """Return the name of the bucket.""" + pass + @abstractmethod def upload( self, From c380a6d9f05622f0f24e2d90440a295eb8cb38cd Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 24 Dec 2024 00:54:09 +0100 Subject: [PATCH 33/33] add `aws.Bucket.rename` method and e2e tests --- src/unicloud/aws/aws.py | 59 ++++++++++++++++++++++++++++++++++++ tests/aws/test_aws_bucket.py | 35 +++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/src/unicloud/aws/aws.py b/src/unicloud/aws/aws.py index a45412c..142d000 100644 --- a/src/unicloud/aws/aws.py +++ b/src/unicloud/aws/aws.py @@ -398,3 +398,62 @@ def file_exists(self, bucket_path: str) -> bool: """ objs = list(self.bucket.objects.filter(Prefix=bucket_path)) return len(objs) > 0 and objs[0].key == bucket_path + + def rename(self, old_path: str, new_path: str): + """ + Rename a file or directory in the S3 bucket. + + This operation renames a file or directory by copying the content to a new path + and then deleting the original path. + + Parameters + ---------- + old_path : str + The current path of the file or directory in the bucket. + new_path : str + The new path for the file or directory in the bucket. + + Raises + ------ + ValueError + If the source file or directory does not exist. + If the destination path already exists. + + Notes + ----- + - For directories, all files and subdirectories are renamed recursively. + - The operation is atomic for individual files but not for directories. + + Examples + -------- + Create the S3 client and bucket: + >>> s3 = S3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, REGION) # doctest: +SKIP + >>> bucket = s3.get_bucket("my-bucket") # doctest: +SKIP + + Rename a file: + >>> bucket.rename("bucket/old_file.txt", "bucket/new_file.txt") # doctest: +SKIP + + Rename a directory: + >>> bucket.rename("bucket/old_dir/", "bucket/new_dir/") # doctest: +SKIP + """ + # Check if the old path exists + objects = list(self.bucket.objects.filter(Prefix=old_path)) + if not objects: + raise ValueError(f"The path '{old_path}' does not exist in the bucket.") + + # Check if the new path already exists + if any(self.bucket.objects.filter(Prefix=new_path)): + raise ValueError(f"The destination path '{new_path}' already exists.") + + # Perform the rename + for obj in objects: + source_key = obj.key + if old_path.endswith("/") and not source_key.startswith(old_path): + continue # Skip unrelated files + destination_key = source_key.replace(old_path, new_path, 1) + self.bucket.Object(destination_key).copy_from( + CopySource={"Bucket": self.bucket.name, "Key": source_key} + ) + obj.delete() + + print(f"Renamed '{old_path}' to '{new_path}'.") diff --git a/tests/aws/test_aws_bucket.py b/tests/aws/test_aws_bucket.py index 333d90f..61297fa 100644 --- a/tests/aws/test_aws_bucket.py +++ b/tests/aws/test_aws_bucket.py @@ -143,6 +143,41 @@ def test_download_empty_directory(self): with pytest.raises(ValueError, match="Directory .* is empty."): self.bucket.download("empty-dir/", "tests/data/empty-dir/") + def test_rename_file(self, test_file: Path): + """ + Test renaming a single file in the bucket. + """ + old_name = "test-rename-old-file.txt" + new_name = "test-rename-new-file.txt" + self.bucket.upload(test_file, old_name, overwrite=True) + + self.bucket.rename(old_name, new_name) + + # Verify the new file exists and the old file does not + assert self.bucket.file_exists(new_name) + assert not self.bucket.file_exists(old_name) + self.bucket.delete(new_name) + + def test_rename_directory(self, upload_test_data: Dict[str, Path]): + """ + Test renaming a directory in the bucket. + """ + old_dir = "old_directory/" + new_dir = "new_directory/" + local_dir = upload_test_data["local_dir"] + self.bucket.upload(local_dir, old_dir, overwrite=True) + + # Rename the directory + self.bucket.rename(old_dir, new_dir) + + # Verify files under the new directory exist and old directory does not + for file in upload_test_data["expected_files"]: + new_file = file.replace("upload-dir", "new_directory") + assert self.bucket.file_exists(new_file) + assert not self.bucket.file_exists(file) + + self.bucket.delete(new_dir) + class TestPropertiesMock: