From fe89b376433131fc48200f5088bbc9efb34fe3e1 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:35:00 +0100 Subject: [PATCH 01/33] add boto3 package for amazon s3 server connection --- environment.yml | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 0f5ecb0..3c8b34e 100644 --- a/environment.yml +++ b/environment.yml @@ -9,6 +9,7 @@ dependencies: - pandas >=1.4.4 - ecmwf-api-client >=1.6.3 - earthengine-api >=0.1.324 + - boto3 >=1.26.50 - joblib >=1.2.0 - pyramids >=0.2.12 - loguru >=0.6.0 diff --git a/requirements.txt b/requirements.txt index 6c4cffa..5cfc87b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ earthengine-api >=0.1.324 ecmwf-api-client >=1.6.3 +boto3 >=1.26.50 gdal >=3.5.3 joblib >=1.2.0 loguru >=0.6.0 From 0a0434c2a37f823b4c3e0b687440b6f4c4ee0bae Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:36:43 +0100 Subject: [PATCH 02/33] add parameters to abstract class methods and add init method to the catalog class --- earth2observe/abstractdatasource.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index f28ebea..61fd400 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -48,17 +48,17 @@ def __init__( pass @abstractmethod - def check_input_dates(self): + def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: str): """Check validity of input dates.""" pass @abstractmethod - def initialize(self): + def initialize(self, *args, **kwargs): """Initialize connection with the data source server (for non ftp servers)""" pass @abstractmethod - def create_grid(self): + def create_grid(self, lat_lim: list, lon_lim: list): """create a grid from the lat/lon boundaries.""" pass @@ -84,6 +84,8 @@ def API(self): class AbstractCatalog(ABC): """abstrach class for the datasource catalog.""" + def __init__(self): + self.catalog = self.get_catalog() @abstractmethod def get_catalog(self): """read the catalog of the datasource from disk or retrieve it from server.""" @@ -92,4 +94,4 @@ def get_catalog(self): @abstractmethod def get_variable(self, var_name) -> Dict[str, str]: """get the details of a specific variable.""" - pass + return self.catalog.get(var_name) From bc4b2bd4781abdea1054d99e5a125a3fa52a04a6 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:38:08 +0100 Subject: [PATCH 03/33] move lines to the abstract class and use super to call the methods --- earth2observe/chirps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index 29b554e..4194670 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -397,14 +397,14 @@ class Catalog(AbstractCatalog): """CHIRPS data catalog.""" def __init__(self): - self.catalog = self.get_catalog() + super().__init__() def get_catalog(self): """return the catalog.""" return { "Precipitation": { - "descriptions": "rainfall [mm/temporal_resolution step]", - "units": "mm/temporal_resolution step", + "descriptions": "rainfall [mm/temporal_resolution]", + "units": "mm/temporal_resolution", "temporal resolution": ["daily", "monthly"], "file name": "rainfall", "var_name": "R", @@ -413,4 +413,4 @@ def get_catalog(self): def get_variable(self, var_name): """get the details of a specific variable.""" - return self.catalog.get(var_name) + return super().get_variable(var_name) From 5f71dd40d426dbf702c25c52d4f69cd9c7a73207 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:38:52 +0100 Subject: [PATCH 04/33] move lines to the abstract class and use super to call the methods --- earth2observe/ecmwf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 8c2dce4..59e0110 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -171,7 +171,7 @@ def download( var_info = catalog.get_variable(var) self.downloadDataset( var_info, dataset=dataset, progress_bar=progress_bar - ) # CaseParameters=[SumMean, Min, Max] + ) # delete the downloaded netcdf del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc") os.remove(del_ecmwf_dataset) @@ -209,6 +209,8 @@ def downloadDataset( """ # Create the directory out_dir = f"{self.path}/{self.temporal_resolution}/{var_info.get('file name')}" + # out_dir = f"{self.path}/{self.temporal_resolution}/{var_info.get('file name')}" + # files = f"{out_dir}/{self.temporal_resolution}_{var_info.get('file name')}" if not os.path.exists(out_dir): os.makedirs(out_dir) @@ -505,10 +507,10 @@ def post_download( class Catalog(AbstractCatalog): """ECMWF data catalog This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc.""" - def __init__(self, version: int = 1): + def __init__(self): # get the catalog - self.catalog = self.get_catalog() - self.version = version + super().__init__() + pass def get_catalog(self): """readthe data catalog from disk.""" @@ -518,7 +520,7 @@ def get_catalog(self): def get_variable(self, var_name): """retrieve a variable form the datasource catalog.""" - return self.catalog.get(var_name) + pass class AuthenticationError(Exception): From 2485421653d1c0da9c782ca13dbf08c2c2e0b7ce Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:41:15 +0100 Subject: [PATCH 05/33] add s3 module and example file --- earth2observe/s3.py | 395 ++++++++++++++++++++++++++++++++++++++++ examples/s3_er5_data.py | 31 ++++ 2 files changed, 426 insertions(+) create mode 100644 earth2observe/s3.py create mode 100644 examples/s3_er5_data.py diff --git a/earth2observe/s3.py b/earth2observe/s3.py new file mode 100644 index 0000000..9cde2ef --- /dev/null +++ b/earth2observe/s3.py @@ -0,0 +1,395 @@ +"""Amazon S3""" +import os +from typing import Dict, List +from pathlib import Path +import datetime as dt +import pandas as pd +import boto3 +import botocore +from botocore import exceptions +from earth2observe.utils import print_progress_bar +from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource + +class S3(AbstractDataSource): + """ + Amazon S3 data source. + """ + def __init__( + self, + temporal_resolution: str = "daily", + start: str = None, + end: str = None, + path: str = "", + variables: list = "precipitation_amount_1hour_Accumulation", + lat_lim: list = None, + lon_lim: list = None, + fmt: str = "%Y-%m-%d", + ): + """S3. + + Parameters + ---------- + temporal_resolution (str, optional): + 'daily' or 'monthly'. Defaults to 'daily'. + start (str, optional): + [description]. Defaults to ''. + end (str, optional): + [description]. Defaults to ''. + path (str, optional): + Path where you want to save the downloaded data. Defaults to ''. + variables (list, optional): + Variable code: VariablesInfo('day').descriptions.keys(). Defaults to []. + lat_lim (list, optional): + [ymin, ymax] (values must be between -50 and 50). Defaults to []. + lon_lim (list, optional): + [xmin, xmax] (values must be between -180 and 180). Defaults to []. + fmt (str, optional): + [description]. Defaults to "%Y-%m-%d". + """ + super().__init__( + start=start, + end=end, + variables=variables, + temporal_resolution=temporal_resolution, + lat_lim=lat_lim, + lon_lim=lon_lim, + fmt=fmt, + ) + self.output_folder = Path(path).absolute() + + # make directory if it not exists + if not os.path.exists(self.output_folder): + os.makedirs(self.output_folder) + + def initialize(self, bucket: str = 'era5-pds') -> object: + """initialize connection with amazon s3 and create a client. + + Parameters + ---------- + bucket: [str] + S3 bucket name. + + Returns + ------- + client: [botocore.client.S3] + Amazon S3 client + """ + # AWS access / secret keys required + # s3 = boto3.resource('s3') + # bucket = s3.Bucket(era5_bucket) + + # No AWS keys required + client = boto3.client('s3', config=botocore.client.Config(signature_version=botocore.UNSIGNED)) + self.client = client + return client + + def create_grid(self, lat_lim: list, lon_lim: list): + """TODO:""" + pass + + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): + """check validity of input dates. + + Parameters + ---------- + temporal_resolution: (str, optional) + [description]. Defaults to 'daily'. + start: (str, optional) + [description]. Defaults to ''. + end: (str, optional) + [description]. Defaults to ''. + fmt: (str, optional) + [description]. Defaults to "%Y-%m-%d". + """ + self.start = dt.datetime.strptime(start, fmt) + self.end = dt.datetime.strptime(end, fmt) + + # Set required data for the daily option + if temporal_resolution == "daily": + self.dates = pd.date_range(self.start, self.end, freq="D") + elif temporal_resolution == "monthly": + self.dates = pd.date_range(self.start, self.end, freq="MS") + + def download(self): + """Download wrapper over all given variables. + + ECMWF method downloads ECMWF daily data for a given variable, temporal_resolution + interval, and spatial extent. + + + Parameters + ---------- + progress_bar : TYPE, optional + 0 or 1. to display the progress bar + dataset:[str] + Default is "interim" + + Returns + ------- + None. + """ + catalog = Catalog() + for var in self.vars: + var_info = catalog.get_variable(var) + var_s3_name = var_info.get("bucket_name") + self.downloadDataset( + var_s3_name + ) + + + def downloadDataset(self, var: str, progress_bar: bool = True): + """Download a climate variable. + + This function downloads ECMWF six-hourly, daily or monthly data. + + Parameters + ---------- + var: [str] + variable detailed information + >>> { + >>> 'descriptions': 'Evaporation [m of water]', + >>> 'units': 'mm', + >>> 'types': 'flux', + >>> 'temporal resolution': ['six hours', 'daily', 'monthly'], + >>> 'file name': 'Evaporation', + >>> } + progress_bar: [bool] + True if you want to display a progress bar. + """ + if progress_bar: + total_amount = len(self.dates) + amount = 0 + print_progress_bar( + amount, + total_amount, + prefix="Progress:", + suffix="Complete", + length=50, + ) + + for date in self.dates: + year = date.strftime("%Y") + month = date.strftime("%m") + # file path patterns for remote S3 objects and corresponding local file + s3_data_key = f'{year}/{month}/data/{var}.nc' + downloaded_file_dir = f'{self.output_folder}/{year}{month}_{self.temporal_resolution}_{var}.nc' + + self.API(s3_data_key, downloaded_file_dir) + + if progress_bar: + amount = amount + 1 + print_progress_bar( + amount, + total_amount, + prefix="Progress:", + suffix="Complete", + length=50, + ) + + def API(self, s3_file_path: str, local_dir_fname: str, bucket: str = 'era5-pds'): + """Download file from s3 bucket. + + Parameters + ---------- + s3_file_path: str + the whole path for the file inside the bcket. i.e. "2022/02/main.nc" + local_dir_fname: [str] + absolute path for the file name and directory in your local drive. + bucket: [str] + bucket name. Default is "era5-pds" + + Returns + ------- + Download the file to your local drive. + """ + if not os.path.isfile(local_dir_fname): # check if file already exists + print(f"Downloading {s3_file_path} from S3...") + try: + self.client.download_file(bucket, s3_file_path, local_dir_fname) + except exceptions.ClientError: + print(f"Error while downloading the {s3_file_path} please check the file name") + else: + print(f"The file {local_dir_fname} already in your local directory") + + + @staticmethod + def parse_response_metadata(response: Dict[str, str]): + """parse client response + + Parameters + ---------- + response: + >>> { + >>> 'RequestId': 'E01V3VYXERE4JV8Z', + >>> 'HostId': 'Nb33s2L/qsQ/oMb8tyT737ymO5sNnAv+KeKEbhvjILbDPvUi0sDVj6zbuRPh/kpmK5BY6Y3EK/A=', + >>> 'HTTPStatusCode': 200, + >>> 'HTTPHeaders': {'x-amz-id-2': 'Nb33s2L/qsQ/oMb8tyT737ymO5sNnAv+KeKEbhvjILbDPvUi0sDVj6zbuRPh/kpmK5BY6Y3EK/A=', + >>> 'x-amz-request-id': 'E01V3VYXERE4JV8Z', + >>> 'date': 'Sun, 15 Jan 2023 22:36:28 GMT', + >>> 'x-amz-bucket-region': 'us-east-1', + >>> 'content-type': 'application/xml', + >>> 'transfer-encoding': 'chunked', + >>> 'server': 'AmazonS3'}, + >>> 'RetryAttempts': 0 + >>> } + """ + response_meta = response.get('ResponseMetadata') + keys = [] + if response_meta.get('HTTPStatusCode') == 200: + contents_list = response.get('Contents') + if contents_list is None: + print(f"No objects are available") # {date.strftime('%B, %Y')} + else: + for obj in contents_list: + keys.append(obj.get('Key')) + print(f"There are {len(keys)} objects available for\n--") # {date.strftime('%B, %Y')} + for k in keys: + print(k) + else: + print("There was an error with your request.") + + return keys + + + +class Catalog(AbstractCatalog): + """S3 data catalog""" + def __init__(self, bucket: str = 'era5-pds'): + """ + + Parameters + ---------- + bucket : + """ + super().__init__() + self.client = self.initialize(bucket=bucket) + + @staticmethod + def initialize(bucket: str = 'era5-pds') -> object: + """initialize connection with amazon s3 and create a client. + + Parameters + ---------- + bucket: [str] + S3 bucket name. + + Returns + ------- + client: [botocore.client.S3] + Amazon S3 client + """ + # AWS access / secret keys required + # s3 = boto3.resource('s3') + # bucket = s3.Bucket(era5_bucket) + + # No AWS keys required + client = boto3.client('s3', config=botocore.client.Config(signature_version=botocore.UNSIGNED)) + return client + + def get_catalog(self): + """return the catalog.""" + return {"precipitation":{ + "descriptions": "rainfall [mm/temporal_resolution]", + "units": "mm/temporal_resolution", + "temporal resolution": ["daily", "monthly"], + "file name": "rainfall", + "var_name": "R", + "bucket_name": "precipitation_amount_1hour_Accumulation" + } + + } + def get_variable(self, var_name) -> Dict[str, str]: + """get the details of a specific variable.""" + return super().get_variable(var_name) + + + def get_available_years(self, bucket: str = 'era5-pds'): + """ + The ERA5 data is chunked into distinct NetCDF files per variable, each containing a month of hourly data. These files are organized in the S3 bucket by year, month, and variable name. + + The data is structured as follows: + + /{year}/{month}/main.nc + /data/{var1}.nc + /{var2}.nc + /{....}.nc + /{varN}.nc + + - where year is expressed as four digits (e.g. YYYY) and month as two digits (e.g. MM). + + Parameters + ---------- + bucket: [str] + S3 bucket name + + Returns + ------- + List: + list of years that have available data. + """ + paginator = self.client.get_paginator('list_objects') + result = paginator.paginate(Bucket=bucket, Delimiter='/') + # for prefix in result.search('CommonPrefixes'): + # print(prefix.get('Prefix')) + years = [i.get('Prefix')[:-1] for i in result.search('CommonPrefixes')] + return years + + + def get_available_data( + self, + date: str, + bucket: str = 'era5-pds', + fmt: str = "%Y-%m-%d", + absolute_path: bool = False, + ) -> List[str]: + """get the available data at a given year + + - Granule variable structure and metadata attributes are stored in main.nc. This file contains coordinate and + auxiliary variable data. This file is also annotated using NetCDF CF metadata conventions. + + Parameters + ---------- + date: [str] + date i.e. "YYYY-mm-dd" + bucket: [str] + The bucket you want to get its available data. Default is 'era5-pds'. + fmt: [str] + Date format. Default is "%Y-%m-%d". + absolute_path: [bool] + True if you want to get the file names including the whole path inside the bucket. + Default is False. + >>> absolute_path = True + [ + '2022/05/air_pressure_at_mean_sea_level.nc', + '2022/05/air_temperature_at_2_metres.nc', + '2022/05/air_temperature_at_2_metres_1hour_Maximum.nc', + '2022/05/air_temperature_at_2_metres_1hour_Minimum.nc', + '2022/05/dew_point_temperature_at_2_metres.nc', + '2022/05/eastward_wind_at_100_metres.nc' + ] + >>> absolute_path = False + [ + 'air_pressure_at_mean_sea_level.nc', + 'air_temperature_at_2_metres.nc', + 'air_temperature_at_2_metres_1hour_Maximum.nc', + 'air_temperature_at_2_metres_1hour_Minimum.nc', + 'dew_point_temperature_at_2_metres.nc', + 'eastward_wind_at_100_metres.nc' + ] + Returns + ------- + List: + available data in a list + """ + date_obj = dt.datetime.strptime(date, fmt) + # date = dt.date(2022,5,1) # update to desired date + prefix = date_obj.strftime('%Y/%m/') + response = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix) + keys = S3.parse_response_metadata(response) + if absolute_path: + available_date = keys + else: + available_date = [i.split("/")[-1] for i in keys] + return available_date diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py new file mode 100644 index 0000000..b5678f5 --- /dev/null +++ b/examples/s3_er5_data.py @@ -0,0 +1,31 @@ +import os +from earth2observe.s3 import S3, Catalog +#%% +s3_catalog = Catalog() +print(s3_catalog.catalog) +s3_catalog.get_variable("precipitation") +years = s3_catalog.get_available_years() +date = "2022-05-01" +# available_date_abs_path = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=True) +# available_date = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=False) +#%% +start = "2009-01-01" +end = "2009-05-01" +time = "monthly" +lat = [4.190755, 4.643963] +lon = [-75.649243, -74.727286] +variables = ["precipitation"] +rpath = os.getcwd() +path = rf"{rpath}/examples/data/s3/era5" + +s3_era5 = S3( + temporal_resolution=time, + start=start, + end=end, + path=path, + variables=variables, + lat_lim=lat, + lon_lim=lon, +) +#%% +s3_era5.download() \ No newline at end of file From 8076b496a80e936912f0c4e979381195dc961a3f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:50:59 +0100 Subject: [PATCH 06/33] make get_variable calls the super class --- earth2observe/ecmwf.py | 2 +- examples/chirps_data.py | 5 ++++- examples/ecmwf_data.py | 6 ++++-- tests/test_earth2observe.py | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 59e0110..18315b2 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -520,7 +520,7 @@ def get_catalog(self): def get_variable(self, var_name): """retrieve a variable form the datasource catalog.""" - pass + return super().get_variable(var_name) class AuthenticationError(Exception): diff --git a/examples/chirps_data.py b/examples/chirps_data.py index d717b82..c046945 100644 --- a/examples/chirps_data.py +++ b/examples/chirps_data.py @@ -1,5 +1,8 @@ -from earth2observe.chirps import CHIRPS +from earth2observe.chirps import CHIRPS, Catalog +#%% +chirps_catalog = Catalog() +print(chirps_catalog.catalog) # %% precipitation start = "2009-01-01" end = "2009-01-10" diff --git a/examples/ecmwf_data.py b/examples/ecmwf_data.py index d9979e4..0a246d8 100644 --- a/examples/ecmwf_data.py +++ b/examples/ecmwf_data.py @@ -19,8 +19,10 @@ # Temperature, Evapotranspiration variables = ["T", "E"] #%% -Vars = Catalog() -print(Vars.catalog) +var = "T" +catalog = Catalog() +print(catalog.catalog) +catalog.get_variable(var) #%% Temperature start = "2009-01-01" end = "2009-02-01" diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 44656c7..6651cd9 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -83,7 +83,7 @@ def test_ecmwf_data_source_instantiate_object( assert e2o.datasource.vars == ecmwf_variables return e2o - def test_download_chirps_backend( + def test_download_ecmwf_backend( self, test_ecmwf_data_source_instantiate_object: CHIRPS, ecmwf_data_source_output_dir: str, From 77219b21c4bd53a9be90bdc14b6fcb191895ac3f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:54:56 +0100 Subject: [PATCH 07/33] run pre-commit --- .gitignore | 2 +- earth2observe/abstractdatasource.py | 5 +- earth2observe/ecmwf.py | 4 +- earth2observe/s3.py | 112 ++++++++++++++-------------- examples/s3_er5_data.py | 4 +- requirements.txt | 2 +- 6 files changed, 68 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 60465f6..39cd94c 100644 --- a/.gitignore +++ b/.gitignore @@ -150,4 +150,4 @@ conda/ .idea/* examples/data/chirps/* *.xml -tests/mo/* \ No newline at end of file +tests/mo/* diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index 61fd400..584a4de 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -48,7 +48,9 @@ def __init__( pass @abstractmethod - def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: str): + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): """Check validity of input dates.""" pass @@ -86,6 +88,7 @@ class AbstractCatalog(ABC): def __init__(self): self.catalog = self.get_catalog() + @abstractmethod def get_catalog(self): """read the catalog of the datasource from disk or retrieve it from server.""" diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 18315b2..fec8653 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -169,9 +169,7 @@ def download( f"Download ECMWF {var} data for period {self.start} till {self.end}" ) var_info = catalog.get_variable(var) - self.downloadDataset( - var_info, dataset=dataset, progress_bar=progress_bar - ) + self.downloadDataset(var_info, dataset=dataset, progress_bar=progress_bar) # delete the downloaded netcdf del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc") os.remove(del_ecmwf_dataset) diff --git a/earth2observe/s3.py b/earth2observe/s3.py index 9cde2ef..4e4f6e3 100644 --- a/earth2observe/s3.py +++ b/earth2observe/s3.py @@ -1,19 +1,21 @@ -"""Amazon S3""" +"""Amazon S3.""" +import datetime as dt import os -from typing import Dict, List from pathlib import Path -import datetime as dt -import pandas as pd +from typing import Dict, List + import boto3 import botocore +import pandas as pd from botocore import exceptions -from earth2observe.utils import print_progress_bar + from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource +from earth2observe.utils import print_progress_bar + class S3(AbstractDataSource): - """ - Amazon S3 data source. - """ + """Amazon S3 data source.""" + def __init__( self, temporal_resolution: str = "daily", @@ -61,7 +63,7 @@ def __init__( if not os.path.exists(self.output_folder): os.makedirs(self.output_folder) - def initialize(self, bucket: str = 'era5-pds') -> object: + def initialize(self, bucket: str = "era5-pds") -> object: """initialize connection with amazon s3 and create a client. Parameters @@ -79,7 +81,9 @@ def initialize(self, bucket: str = 'era5-pds') -> object: # bucket = s3.Bucket(era5_bucket) # No AWS keys required - client = boto3.client('s3', config=botocore.client.Config(signature_version=botocore.UNSIGNED)) + client = boto3.client( + "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) + ) self.client = client return client @@ -134,10 +138,7 @@ def download(self): for var in self.vars: var_info = catalog.get_variable(var) var_s3_name = var_info.get("bucket_name") - self.downloadDataset( - var_s3_name - ) - + self.downloadDataset(var_s3_name) def downloadDataset(self, var: str, progress_bar: bool = True): """Download a climate variable. @@ -173,8 +174,8 @@ def downloadDataset(self, var: str, progress_bar: bool = True): year = date.strftime("%Y") month = date.strftime("%m") # file path patterns for remote S3 objects and corresponding local file - s3_data_key = f'{year}/{month}/data/{var}.nc' - downloaded_file_dir = f'{self.output_folder}/{year}{month}_{self.temporal_resolution}_{var}.nc' + s3_data_key = f"{year}/{month}/data/{var}.nc" + downloaded_file_dir = f"{self.output_folder}/{year}{month}_{self.temporal_resolution}_{var}.nc" self.API(s3_data_key, downloaded_file_dir) @@ -188,7 +189,7 @@ def downloadDataset(self, var: str, progress_bar: bool = True): length=50, ) - def API(self, s3_file_path: str, local_dir_fname: str, bucket: str = 'era5-pds'): + def API(self, s3_file_path: str, local_dir_fname: str, bucket: str = "era5-pds"): """Download file from s3 bucket. Parameters @@ -209,14 +210,15 @@ def API(self, s3_file_path: str, local_dir_fname: str, bucket: str = 'era5-pds') try: self.client.download_file(bucket, s3_file_path, local_dir_fname) except exceptions.ClientError: - print(f"Error while downloading the {s3_file_path} please check the file name") + print( + f"Error while downloading the {s3_file_path} please check the file name" + ) else: print(f"The file {local_dir_fname} already in your local directory") - @staticmethod def parse_response_metadata(response: Dict[str, str]): - """parse client response + """parse client response. Parameters ---------- @@ -235,16 +237,18 @@ def parse_response_metadata(response: Dict[str, str]): >>> 'RetryAttempts': 0 >>> } """ - response_meta = response.get('ResponseMetadata') + response_meta = response.get("ResponseMetadata") keys = [] - if response_meta.get('HTTPStatusCode') == 200: - contents_list = response.get('Contents') + if response_meta.get("HTTPStatusCode") == 200: + contents_list = response.get("Contents") if contents_list is None: - print(f"No objects are available") # {date.strftime('%B, %Y')} + print("No objects are available") # {date.strftime('%B, %Y')} else: for obj in contents_list: - keys.append(obj.get('Key')) - print(f"There are {len(keys)} objects available for\n--") # {date.strftime('%B, %Y')} + keys.append(obj.get("Key")) + print( + f"There are {len(keys)} objects available for\n--" + ) # {date.strftime('%B, %Y')} for k in keys: print(k) else: @@ -253,10 +257,10 @@ def parse_response_metadata(response: Dict[str, str]): return keys - class Catalog(AbstractCatalog): - """S3 data catalog""" - def __init__(self, bucket: str = 'era5-pds'): + """S3 data catalog.""" + + def __init__(self, bucket: str = "era5-pds"): """ Parameters @@ -267,7 +271,7 @@ def __init__(self, bucket: str = 'era5-pds'): self.client = self.initialize(bucket=bucket) @staticmethod - def initialize(bucket: str = 'era5-pds') -> object: + def initialize(bucket: str = "era5-pds") -> object: """initialize connection with amazon s3 and create a client. Parameters @@ -285,29 +289,30 @@ def initialize(bucket: str = 'era5-pds') -> object: # bucket = s3.Bucket(era5_bucket) # No AWS keys required - client = boto3.client('s3', config=botocore.client.Config(signature_version=botocore.UNSIGNED)) + client = boto3.client( + "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) + ) return client def get_catalog(self): """return the catalog.""" - return {"precipitation":{ - "descriptions": "rainfall [mm/temporal_resolution]", - "units": "mm/temporal_resolution", - "temporal resolution": ["daily", "monthly"], - "file name": "rainfall", - "var_name": "R", - "bucket_name": "precipitation_amount_1hour_Accumulation" + return { + "precipitation": { + "descriptions": "rainfall [mm/temporal_resolution]", + "units": "mm/temporal_resolution", + "temporal resolution": ["daily", "monthly"], + "file name": "rainfall", + "var_name": "R", + "bucket_name": "precipitation_amount_1hour_Accumulation", + } } - } def get_variable(self, var_name) -> Dict[str, str]: """get the details of a specific variable.""" return super().get_variable(var_name) - - def get_available_years(self, bucket: str = 'era5-pds'): - """ - The ERA5 data is chunked into distinct NetCDF files per variable, each containing a month of hourly data. These files are organized in the S3 bucket by year, month, and variable name. + def get_available_years(self, bucket: str = "era5-pds"): + """The ERA5 data is chunked into distinct NetCDF files per variable, each containing a month of hourly data. These files are organized in the S3 bucket by year, month, and variable name. The data is structured as follows: @@ -329,22 +334,21 @@ def get_available_years(self, bucket: str = 'era5-pds'): List: list of years that have available data. """ - paginator = self.client.get_paginator('list_objects') - result = paginator.paginate(Bucket=bucket, Delimiter='/') + paginator = self.client.get_paginator("list_objects") + result = paginator.paginate(Bucket=bucket, Delimiter="/") # for prefix in result.search('CommonPrefixes'): # print(prefix.get('Prefix')) - years = [i.get('Prefix')[:-1] for i in result.search('CommonPrefixes')] + years = [i.get("Prefix")[:-1] for i in result.search("CommonPrefixes")] return years - def get_available_data( - self, - date: str, - bucket: str = 'era5-pds', - fmt: str = "%Y-%m-%d", - absolute_path: bool = False, + self, + date: str, + bucket: str = "era5-pds", + fmt: str = "%Y-%m-%d", + absolute_path: bool = False, ) -> List[str]: - """get the available data at a given year + """get the available data at a given year. - Granule variable structure and metadata attributes are stored in main.nc. This file contains coordinate and auxiliary variable data. This file is also annotated using NetCDF CF metadata conventions. @@ -385,7 +389,7 @@ def get_available_data( """ date_obj = dt.datetime.strptime(date, fmt) # date = dt.date(2022,5,1) # update to desired date - prefix = date_obj.strftime('%Y/%m/') + prefix = date_obj.strftime("%Y/%m/") response = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix) keys = S3.parse_response_metadata(response) if absolute_path: diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py index b5678f5..e4178ec 100644 --- a/examples/s3_er5_data.py +++ b/examples/s3_er5_data.py @@ -1,5 +1,7 @@ import os + from earth2observe.s3 import S3, Catalog + #%% s3_catalog = Catalog() print(s3_catalog.catalog) @@ -28,4 +30,4 @@ lon_lim=lon, ) #%% -s3_era5.download() \ No newline at end of file +s3_era5.download() diff --git a/requirements.txt b/requirements.txt index 5cfc87b..97fc98f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ +boto3 >=1.26.50 earthengine-api >=0.1.324 ecmwf-api-client >=1.6.3 -boto3 >=1.26.50 gdal >=3.5.3 joblib >=1.2.0 loguru >=0.6.0 From 99d1e3772098d186e5b5a5932517a7fa58901e24 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 03:57:08 +0100 Subject: [PATCH 08/33] add .gitignore file to the download folders --- examples/data/s3/.gitignore | 7 +++++++ tests/data/delete/.gitignore | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 examples/data/s3/.gitignore create mode 100644 tests/data/delete/.gitignore diff --git a/examples/data/s3/.gitignore b/examples/data/s3/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/examples/data/s3/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/.gitignore b/tests/data/delete/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore From cedff64f36b66a90e659355cb1d7683360e8c51e Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 19:53:19 +0100 Subject: [PATCH 09/33] add serapeum_utils --- conda-lock.yml | 459 ++++++++++++++++++++++++++++++++++------------- environment.yml | 1 + requirements.txt | 1 + 3 files changed, 333 insertions(+), 128 deletions(-) diff --git a/conda-lock.yml b/conda-lock.yml index 8d01e6f..2cc0aea 100644 --- a/conda-lock.yml +++ b/conda-lock.yml @@ -15,9 +15,9 @@ metadata: - url: conda-forge used_env_vars: [] content_hash: - linux-64: 254a8ec847efd0a02a050ea9fc62f1a75932bb3645fa638fc79971d4b8f35430 - osx-64: d8e507a190f5b7c996b48cd0a82bc6c4ed5fed5a8279ee8d4157cb3c39be60c3 - win-64: cc58bac7c0731eff5cf8ea05e67e60e4ddf69f70dfdf9d0080a8a7faa1a4d316 + linux-64: 9eaae55afe9701810ae0e852f1b5fe4ebdffd940b4165beba9255cd4bf9ded7f + osx-64: e72b214899b7c355b81e32e383e1a018e3a880ac272dcc35efc0689971140455 + win-64: e6c1578e0dbf9fb2288f4763e9c778a9cfbd2c8d30dc770640394a5340b7e79e platforms: - linux-64 - osx-64 @@ -397,16 +397,16 @@ package: version: '20220623.0' - category: main dependencies: - libgcc-ng: '>=9.4.0' - libstdcxx-ng: '>=9.4.0' + libgcc-ng: '>=12' + libstdcxx-ng: '>=12' hash: - md5: c77f5e4e418fa47d699d6afa54c5d444 - sha256: f7c8866b27c4b6e2b2b84aae544fab539dfbfe5420c2c16fb868e9440bdb001e + md5: 0f683578378cddb223e7fd24f785ab2a + sha256: 4df6a29b71264fb25462065e8cddcf5bca60776b1801974af8cbd26b7425fcda manager: conda name: libaec optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-h9c3ff4c_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda version: 1.0.6 - category: main dependencies: @@ -437,14 +437,14 @@ package: dependencies: libgcc-ng: '>=12' hash: - md5: fc84a0446e4e4fb882e78d786cfb9734 - sha256: 6f7cbc9347964e7f9697bde98a8fb68e0ed926888b3116474b1224eaa92209dc + md5: 5cc781fd91968b11a8a7fdbee0982676 + sha256: f9983a8ea03531f2c14bce76c870ca325c0fddf0c4e872bff1f78bc52624179c manager: conda name: libdeflate optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.17-h0b41bf4_0.conda + version: '1.17' - category: main dependencies: libgcc-ng: '>=7.5.0' @@ -1214,7 +1214,7 @@ package: dependencies: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libgcc-ng: '>=12' libstdcxx-ng: '>=12' libwebp-base: '>=1.2.4,<2.0a0' @@ -1222,13 +1222,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: a01611c54334d783847879ee40109657 - sha256: 7237772229da1058fae73ae6f04ad846551a44d7da602e9d328b81049d3219a2 + md5: 2e648a34072eb39d7c4fc2a9981c5f0c + sha256: e3e18d91fb282b61288d4fd2574dfa31f7ae90ef2737f96722fb6ad3257862ee manager: conda name: libtiff optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h82bc61c_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda version: 4.5.0 - category: main dependencies: @@ -1357,14 +1357,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -1515,6 +1515,18 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: libgcc-ng: '>=12' @@ -1769,14 +1781,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: libgcc-ng: '>=12' @@ -1806,18 +1818,30 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/linux-64/rtree-1.0.1-py310hbdcdc62_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -2877,7 +2901,7 @@ package: kealib: '>=1.5.0,<1.6.0a0' lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libgcc-ng: '>=12' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' @@ -2899,18 +2923,18 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' xerces-c: '>=3.2.4,<3.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: f16dc7517110a11910f7c1d9def32d38 - sha256: 84d5d701b0b739caa7a71eca47be2625079a2dd48c7638b9e5aedb72c916e593 + md5: 0a5408aac806b97ed97ca718957eb4b0 + sha256: c4a9694125c1ed214570df2e75ed1e799fa2aff8e85b365170ecd822d07aca22 manager: conda name: libgdal optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-he31f7c0_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h10cbb15_3.conda version: 3.6.2 - category: main dependencies: @@ -2949,20 +2973,20 @@ package: dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' libgcc-ng: '>=12' - libgdal: 3.6.2 he31f7c0_1 + libgdal: 3.6.2 h10cbb15_3 libstdcxx-ng: '>=12' numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' python_abi: 3.10.* *_cp310 hash: - md5: d6855fd7042a21cba318f68874e5b3e6 - sha256: fca1260c69715a21f12a0d5421495909e5bf09ab2b3662eb12b4ec6da5d871c3 + md5: 8f09d00becd16e21c92f26a99b231aac + sha256: eb3824b695871188e6b3a11f0fe4b950ba79327a6e33b7d69dd23fa85d7b49d6 manager: conda name: gdal optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py310hc1b7723_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py310hc1b7723_3.conda version: 3.6.2 - category: main dependencies: @@ -3008,6 +3032,21 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: attrs: '>=17' @@ -3042,14 +3081,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: branca: '>=0.6.0' @@ -3101,6 +3140,34 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: google-auth: '>=2.14.1,<3.0dev' @@ -3229,17 +3296,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: @@ -3453,14 +3520,14 @@ package: - category: main dependencies: {} hash: - md5: ce2a6075114c9b64ad8cace52492feee - sha256: 0153de9987fa6e8dd5be45920470d579af433d4560bfd77318a72b3fd75fb6dc + md5: e3894420cf8b6abbf6c4d3d9742fbb4a + sha256: b322e190fd6fe631e1f4836ef99cbfb8352c03c30b51cb5baa216f7c9124d82e manager: conda name: libdeflate optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libdeflate-1.14-hb7f2c08_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/osx-64/libdeflate-1.17-hac1461d_0.conda + version: '1.17' - category: main dependencies: {} hash: @@ -3748,15 +3815,15 @@ package: version: '20220623.0' - category: main dependencies: - libcxx: '>=11.1.0' + libcxx: '>=14.0.6' hash: - md5: 0a49b696f11ed805ee4690479cc5e950 - sha256: 5c45ae356d10b6b78a9985e19d4cbd0e71cc76d1b43028f32737ef313ed525de + md5: 7c0f82f435ab4c48d65dc9b28db2ad9e + sha256: 38d32f4c7efddc204e53f43cd910122d3e6a997de1a3cd15f263217b225a9cdf manager: conda name: libaec optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libaec-1.0.6-he49afe7_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/osx-64/libaec-1.0.6-hf0c8a7f_1.conda version: 1.0.6 - category: main dependencies: @@ -4198,19 +4265,19 @@ package: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' libcxx: '>=14.0.6' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libwebp-base: '>=1.2.4,<2.0a0' libzlib: '>=1.2.13,<1.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: f32f9708c8d6f13e20a524c6da9a881e - sha256: 6659b6b71e79976e1bdd4b730a92425954e88cad6a184e274d6523d34d85f10e + md5: 35f714269a801f7c3cb522aacd3c0e69 + sha256: 03d00d6a3b1e569e9a8da66a9ad75a29c9c676dc7de6c16771abbb961abded2c manager: conda name: libtiff optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libtiff-4.5.0-h6268bbc_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/libtiff-4.5.0-hee9004a_2.conda version: 4.5.0 - category: main dependencies: @@ -4331,14 +4398,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -4486,6 +4553,18 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: libcxx: '>=14.0.4' @@ -4743,14 +4822,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -4779,18 +4858,30 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/osx-64/rtree-1.0.1-py310had9ce37_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -5705,7 +5796,7 @@ package: lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' libcxx: '>=14.0.6' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' libnetcdf: '>=4.8.1,<4.8.2.0a0' @@ -5724,18 +5815,18 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' xerces-c: '>=3.2.4,<3.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: 1eb9593ce19fa949f7c32cb289271e82 - sha256: 61d2094cc01664cedcf23ab34616802b63ed31d65c520815977c8999f4f27ee6 + md5: 1ae43833d5a0c8d7504a991eb13e2439 + sha256: f077161cbbfe41d687804237adb169949ed911432e4a23786aab1fa017ef1b4a manager: conda name: libgdal optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libgdal-3.6.2-h44a409f_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/libgdal-3.6.2-h623d8b8_3.conda version: 3.6.2 - category: main dependencies: @@ -5800,19 +5891,19 @@ package: dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' libcxx: '>=14.0.6' - libgdal: 3.6.2 h44a409f_1 + libgdal: 3.6.2 h623d8b8_3 numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' python_abi: 3.10.* *_cp310 hash: - md5: 16d96cc0855982325700c8623af85ffb - sha256: ddebc8a3f9ad251ed8acbfbf0db0cf4af93fdc1264ad5757d64baa586a783e86 + md5: eebf4977fcaa1f20f94b55af26110eab + sha256: 043c842b3d96f3aa4ce491f12f8b5510aade83d74566655894f18d81837cfc81 manager: conda name: gdal optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/gdal-3.6.2-py310h5abc6fc_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/gdal-3.6.2-py310h5abc6fc_3.conda version: 3.6.2 - category: main dependencies: @@ -5878,6 +5969,21 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: attrs: '>=17' @@ -5997,14 +6103,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: branca: '>=0.6.0' @@ -6056,6 +6162,34 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: google-auth: '>=2.14.1,<3.0dev' @@ -6183,17 +6317,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: @@ -6580,16 +6714,17 @@ package: version: '20220623.0' - category: main dependencies: - vc: '>=14.1,<15.0a0' - vs2015_runtime: '>=14.16.27012' + ucrt: '>=10.0.20348.0' + vc: '>=14.2,<15' + vs2015_runtime: '>=14.29.30139' hash: - md5: ac78b243f1ee03a2412b6e328aa3a12d - sha256: 47f2ef0486d690b2bc34035a165a86c1edcc18d48f1f8aa8eead594afa0dfebf + md5: f98474a8245f55f4a273889dbe7bf193 + sha256: 441f580f90279bd62bd27fb82d0bbbb2c2d9f850fcc4c8781f199c5287cd1499 manager: conda name: libaec optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libaec-1.0.6-h39d44d4_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/win-64/libaec-1.0.6-h63175ca_1.conda version: 1.0.6 - category: main dependencies: @@ -6620,17 +6755,18 @@ package: version: 1.1.2 - category: main dependencies: + ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' hash: - md5: 4366e00d3270eb229c026920474a6dda - sha256: c8b156fc81006234cf898f933b06bed8bb475970cb7983d0eceaf90db65beb8b + md5: ae9dfb57bcb42093a2417aceabb530f7 + sha256: 76e642ca8a11da1b537506447f8089353b6607956c069c938a4bec4de36e1194 manager: conda name: libdeflate optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libdeflate-1.14-hcfcfb64_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/win-64/libdeflate-1.17-hcfcfb64_0.conda + version: '1.17' - category: main dependencies: vc: '>=14.1,<15.0a0' @@ -7221,14 +7357,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -7397,6 +7533,18 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -7506,7 +7654,7 @@ package: dependencies: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libzlib: '>=1.2.13,<1.3.0a0' ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' @@ -7514,13 +7662,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: d9b568beb74c97e53dbb9531b14f69c0 - sha256: dc13f42b392e05a1e705ceebcde82ed3e15663b511e78585d26fc4b9bf628b59 + md5: 2e003e276cc1375192569c96afd3d984 + sha256: 86cf8066db11f84b506ba246944901584ab199dfe7490586f5e9b6c299e3b8e0 manager: conda name: libtiff optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libtiff-4.5.0-hc4f729c_0.conda + url: https://conda.anaconda.org/conda-forge/win-64/libtiff-4.5.0-hf8721a0_2.conda version: 4.5.0 - category: main dependencies: @@ -7667,14 +7815,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -7706,18 +7854,30 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/win-64/rtree-1.0.1-py310h1cbd46b_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -8845,7 +9005,7 @@ package: kealib: '>=1.5.0,<1.6.0a0' lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' libnetcdf: '>=4.8.1,<4.8.2.0a0' @@ -8864,7 +9024,7 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' @@ -8872,13 +9032,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: 9a4ef5cd8e935b794df7aea0cc4ca70a - sha256: 140e1141816bd5e7a560fef469ca435394d07eea5f9f37d01c18be59cee1a603 + md5: a73eceda4d87ffc2ae1e6d151f3f878a + sha256: 6dcd51c0c30b3c788b207ebc1ce716b27218333a2aa81e130d2e85bf26681d42 manager: conda name: libgdal optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libgdal-3.6.2-hffd0036_1.conda + url: https://conda.anaconda.org/conda-forge/win-64/libgdal-3.6.2-h060c9ed_3.conda version: 3.6.2 - category: main dependencies: @@ -8910,6 +9070,21 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: libblas: '>=3.9.0,<4.0a0' @@ -8937,14 +9112,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: numpy: '>=1.21.6,<2.0a0' @@ -8998,7 +9173,7 @@ package: - category: main dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' - libgdal: 3.6.2 hffd0036_1 + libgdal: 3.6.2 h060c9ed_3 numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' @@ -9007,13 +9182,13 @@ package: vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' hash: - md5: ed437558c068417a3fedd81f51aaa828 - sha256: 6c778b866106fe1cbe7560982705ad4f280fe42b1fb9dc290cd4b978c535a55c + md5: 8934913fb065c580d1d78b55fd1e9b42 + sha256: a40a315a226c12cc1eace77d58220f9120f37027d17fa2f8698fd937fe95a725 manager: conda name: gdal optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/gdal-3.6.2-py310h644bc08_1.conda + url: https://conda.anaconda.org/conda-forge/win-64/gdal-3.6.2-py310h644bc08_3.conda version: 3.6.2 - category: main dependencies: @@ -9069,6 +9244,19 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 - category: main dependencies: geos: '>=3.11.1,<3.11.2.0a0' @@ -9101,6 +9289,21 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/snuggs-1.4.7-py_0.tar.bz2 version: 1.4.7 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: attrs: '>=17' @@ -9349,17 +9552,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: diff --git a/environment.yml b/environment.yml index 3c8b34e..034ab3d 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - boto3 >=1.26.50 - joblib >=1.2.0 - pyramids >=0.2.12 + - serapeum_utils >=0.1.0 - loguru >=0.6.0 - PyYAML >=6.0 - pathlib >=1.0.1 diff --git a/requirements.txt b/requirements.txt index 97fc98f..7ca3bb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,6 @@ pandas >=1.4.4 pathlib >=1.0.1 pip >=22.3.1 pyramids-gis >=0.2.12 +serapeum_utils >=0.1.0 PyYAML >=6.0 requests >=2.28.1 From b369ed14c04ef3047bab492a3cfd95676f3e4704 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 19:57:43 +0100 Subject: [PATCH 10/33] replace utils module by serapeum_utils package --- earth2observe/chirps.py | 2 +- earth2observe/ecmwf.py | 2 +- earth2observe/s3.py | 2 +- earth2observe/utils.py | 45 ----------------------------------------- 4 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 earth2observe/utils.py diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index 4194670..1702658 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -11,7 +11,7 @@ from pyramids.utils import extractFromGZ from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from earth2observe.utils import print_progress_bar +from serapeum_utils.utils import print_progress_bar class CHIRPS(AbstractDataSource): diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index fec8653..6d320b2 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -18,7 +18,7 @@ from earth2observe import __path__ from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from earth2observe.utils import print_progress_bar +from serapeum_utils.utils import print_progress_bar class ECMWF(AbstractDataSource): diff --git a/earth2observe/s3.py b/earth2observe/s3.py index 4e4f6e3..3035af5 100644 --- a/earth2observe/s3.py +++ b/earth2observe/s3.py @@ -10,7 +10,7 @@ from botocore import exceptions from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from earth2observe.utils import print_progress_bar +from serapeum_utils.utils import print_progress_bar class S3(AbstractDataSource): diff --git a/earth2observe/utils.py b/earth2observe/utils.py deleted file mode 100644 index a3697a4..0000000 --- a/earth2observe/utils.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import sys - - -def print_progress_bar( - i: int, - total: int, - prefix: str = "", - suffix: str = "", - decimals: int = 1, - length: int = 100, - fill: str = "â–ˆ", -): - """print_progress_bar. - - Parameters - ---------- - i: [int] - Iteration number - total: [int] - Total iterations - prefix: [str] - Name after bar - suffix: [str] - Decimals of percentage - decimals: [int] - - length: [int] - width of the waitbar - fill: [str] - bar fill - """ - # Adjust when it is a linux computer - if os.name == "posix" and total == 0: - total = 0.0001 - - percent = ("{0:." + str(decimals) + "f}").format(100 * (i / float(total))) - filled = int(length * i // total) - bar = fill * filled + "-" * (length - filled) - - sys.stdout.write("\r%s |%s| %s%% %s" % (prefix, bar, percent, suffix)) - sys.stdout.flush() - - if i == total: - print() From 17d0bf0d6cc7be5a59eb6688affb66f6754b79f8 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 20:50:28 +0100 Subject: [PATCH 11/33] bump up serapeum_utils version --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 034ab3d..f010b70 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - boto3 >=1.26.50 - joblib >=1.2.0 - pyramids >=0.2.12 - - serapeum_utils >=0.1.0 + - serapeum_utils >=0.1.1 - loguru >=0.6.0 - PyYAML >=6.0 - pathlib >=1.0.1 diff --git a/requirements.txt b/requirements.txt index 7ca3bb7..a209c72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ pandas >=1.4.4 pathlib >=1.0.1 pip >=22.3.1 pyramids-gis >=0.2.12 -serapeum_utils >=0.1.0 +serapeum_utils >=0.1.1 PyYAML >=6.0 requests >=2.28.1 From 238d159c92fd97101d3ee52ad80fb8a922bd8401 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 22:32:59 +0100 Subject: [PATCH 12/33] add tests for s3 --- tests/data/delete/.gitignore | 7 ------- tests/test_chirps.py | 4 ++-- tests/test_ecmwf.py | 8 ++++++-- tests/test_s3.py | 40 ++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 11 deletions(-) delete mode 100644 tests/data/delete/.gitignore create mode 100644 tests/test_s3.py diff --git a/tests/data/delete/.gitignore b/tests/data/delete/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/test_chirps.py b/tests/test_chirps.py index ff09f81..61b684d 100644 --- a/tests/test_chirps.py +++ b/tests/test_chirps.py @@ -44,8 +44,8 @@ def test_download( test_create_chirps_object.download() filelist = glob.glob( - os.path.join(f"{chirps_base_dir}/chirps/precipitation", f"{fname}*.tif") + os.path.join(f"{chirps_base_dir}", f"{fname}*.tif") ) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{chirps_base_dir}/chirps/precipitation") + shutil.rmtree(f"{chirps_base_dir}") diff --git a/tests/test_ecmwf.py b/tests/test_ecmwf.py index a1d1c95..d7601ed 100644 --- a/tests/test_ecmwf.py +++ b/tests/test_ecmwf.py @@ -34,7 +34,11 @@ def test_download( number_downloaded_files: int, ): test_create_ecmwf_object.download() - filelist = glob.glob(os.path.join(f"{ecmwf_base_dir}/daily/Evaporation/", f"*.tif")) + filelist = glob.glob(os.path.join(f"{ecmwf_base_dir}", f"*.tif")) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{ecmwf_base_dir}/daily") + try: + shutil.rmtree(f"{ecmwf_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted") + diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 0000000..8ea9421 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,40 @@ +from typing import List +import glob +import os +import shutil +import pytest +from earth2observe.s3 import S3 + +@pytest.fixture(scope="session") +def test_create_s3_object( + monthly_dates: List, + lat_bounds: List, + lon_bounds: List, + s3_era5_base_dir: str, + s3_era5_variables: List[str], +): + Coello = S3( + start=monthly_dates[0], + end=monthly_dates[1], + lat_lim=lat_bounds, + lon_lim=lon_bounds, + path=s3_era5_base_dir, + variables=s3_era5_variables, + ) + assert isinstance(Coello, S3) + return Coello + + +def test_download( + test_create_s3_object: S3, + s3_era5_base_dir: str, + number_downloaded_files: int, +): + test_create_s3_object.download() + filelist = glob.glob(os.path.join(f"{s3_era5_base_dir}", f"*.nc")) + assert len(filelist) == number_downloaded_files + # delete the files + try: + shutil.rmtree(f"{s3_era5_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted") From 870269f6f14eb10b5e2cffed9bffc274334a707b Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 22:52:26 +0100 Subject: [PATCH 13/33] update tests --- earth2observe/abstractdatasource.py | 11 ++++++-- earth2observe/chirps.py | 40 ++++++++++++----------------- earth2observe/earth2observe.py | 3 ++- earth2observe/ecmwf.py | 15 +++-------- earth2observe/s3.py | 14 ++++------ examples/abstract_implementation.py | 23 ++++++++++++++--- examples/chirps_data.py | 4 +-- examples/ecmwf_data.py | 2 +- examples/s3_er5_data.py | 2 +- tests/conftest.py | 36 +++++++++++++++++++++----- 10 files changed, 89 insertions(+), 61 deletions(-) diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index 584a4de..8749a0d 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -1,5 +1,7 @@ -from abc import ABC, abstractmethod +import os from typing import Dict +from pathlib import Path +from abc import ABC, abstractmethod class AbstractDataSource(ABC): @@ -14,7 +16,7 @@ def __init__( lat_lim: list = None, lon_lim: list = None, fmt: str = "%Y-%m-%d", - # path: str = "", + path: str = "", ): """ @@ -45,6 +47,11 @@ def __init__( self.create_grid(lat_lim, lon_lim) self.check_input_dates(start, end, temporal_resolution, fmt) + self.path = Path(path).absolute() + # Create the directory + if not os.path.exists(self.path): + os.makedirs(self.path) + pass @abstractmethod diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index 1702658..d0a770d 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -66,14 +66,8 @@ def __init__( lat_lim=lat_lim, lon_lim=lon_lim, fmt=fmt, + path=path, ) - self.output_folder = os.path.join( - Path(path).absolute(), "chirps", "precipitation" - ) - - # make directory if it not exists - if not os.path.exists(self.output_folder): - os.makedirs(self.output_folder) def check_input_dates( self, start: str, end: str, temporal_resolution: str, fmt: str @@ -105,10 +99,10 @@ def check_input_dates( # Define timestep for the timedates if temporal_resolution.lower() == "daily": self.time_freq = "D" - # self.output_folder = os.path.join(path, "precipitation", "chirps", "daily") + # self.path = os.path.join(path, "precipitation", "chirps", "daily") elif temporal_resolution.lower() == "monthly": self.time_freq = "MS" - # self.output_folder = os.path.join( + # self.path = os.path.join( # path, "Precipitation", "CHIRPS", "Monthly" # ) else: @@ -189,7 +183,7 @@ def download(self, progress_bar: bool = True, cores=None, *args, **kwargs): """ # Pass variables to parallel function and run args = [ - self.output_folder, + self.path, self.temporal_resolution, self.xID, self.yID, @@ -238,7 +232,7 @@ def API(self, date, args): args: [list] """ - [output_folder, temp_resolution, xID, yID, lon_lim, latlim] = args + [path, temp_resolution, xID, yID, lon_lim, latlim] = args # Define FTP path to directory if temp_resolution.lower() == "daily": @@ -252,11 +246,11 @@ def API(self, date, args): if temp_resolution.lower() == "daily": filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif.gz" outfilename = os.path.join( - output_folder, + path, f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) DirFileEnd = os.path.join( - output_folder, + path, f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) elif temp_resolution == "monthly": @@ -264,23 +258,23 @@ def API(self, date, args): f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz" ) outfilename = os.path.join( - output_folder, + path, f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif", ) DirFileEnd = os.path.join( - output_folder, + path, f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) else: raise KeyError("The input temporal_resolution interval is not supported") - self.callAPI(pathFTP, output_folder, filename) + self.callAPI(pathFTP, path, filename) self.post_download( - output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd + path, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd ) @staticmethod - def callAPI(pathFTP: str, output_folder: str, filename: str): + def callAPI(pathFTP: str, path: str, filename: str): """send the request to the server. RetrieveData method retrieves CHIRPS data for a given date from the @@ -289,7 +283,7 @@ def callAPI(pathFTP: str, output_folder: str, filename: str): Parameters ---------- filename - output_folder + path pathFTP @@ -313,14 +307,14 @@ def callAPI(pathFTP: str, output_folder: str, filename: str): ftp.retrlines("LIST", listing.append) # download the global rainfall file - local_filename = os.path.join(output_folder, filename) + local_filename = os.path.join(path, filename) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write, 8192) lf.close() def post_download( self, - output_folder, + path, filename, lon_lim, latlim, @@ -333,7 +327,7 @@ def post_download( Parameters ---------- - output_folder: [str] + path: [str] directory where files will be saved filename: [str] file name @@ -346,7 +340,7 @@ def post_download( """ try: # unzip the file - zip_filename = os.path.join(output_folder, filename) + zip_filename = os.path.join(path, filename) extractFromGZ(zip_filename, outfilename, delete=True) # open tiff file diff --git a/earth2observe/earth2observe.py b/earth2observe/earth2observe.py index 6ed6ab4..841c749 100644 --- a/earth2observe/earth2observe.py +++ b/earth2observe/earth2observe.py @@ -1,12 +1,13 @@ """Front end module that runs each data source backend.""" from earth2observe.chirps import CHIRPS from earth2observe.ecmwf import ECMWF +from earth2observe.s3 import S3 class Earth2Observe: """End user class to call all the data source classes abailable in earth2observe.""" - DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS} + DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS, "amazon-s3": S3} def __init__( self, diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 6d320b2..c6b0778 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -20,7 +20,6 @@ from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource from serapeum_utils.utils import print_progress_bar - class ECMWF(AbstractDataSource): """RemoteSensing. @@ -70,8 +69,8 @@ def __init__( lat_lim=lat_lim, lon_lim=lon_lim, fmt=fmt, + path=path, ) - self.path = Path(path).absolute() def check_input_dates( self, start: str, end: str, temporal_resolution: str, fmt: str @@ -205,18 +204,10 @@ def downloadDataset( progress_bar: [bool] True if you want to display a progress bar. """ - # Create the directory - out_dir = f"{self.path}/{self.temporal_resolution}/{var_info.get('file name')}" - # out_dir = f"{self.path}/{self.temporal_resolution}/{var_info.get('file name')}" - # files = f"{out_dir}/{self.temporal_resolution}_{var_info.get('file name')}" - - if not os.path.exists(out_dir): - os.makedirs(out_dir) - # trigger the request to the server self.API(var_info, dataset) # process the downloaded data - self.post_download(var_info, out_dir, dataset, progress_bar) + self.post_download(var_info, self.path, dataset, progress_bar) def API(self, var_info, dataset): """form the request url abd trigger the request. @@ -482,7 +473,7 @@ def post_download( # Define the out name name_out = os.path.join( out_dir, - f"%{var_output_name}_ECMWF_ERA-Interim_{Var_unit}_{self.temporal_resolution}_{year}.{month}.{day}.tif", + f"{var_output_name}_ECMWF_ERA-Interim_{Var_unit}_{self.temporal_resolution}_{year}.{month}.{day}.tif", ) # Create Tiff files diff --git a/earth2observe/s3.py b/earth2observe/s3.py index 3035af5..a91c3be 100644 --- a/earth2observe/s3.py +++ b/earth2observe/s3.py @@ -18,7 +18,7 @@ class S3(AbstractDataSource): def __init__( self, - temporal_resolution: str = "daily", + temporal_resolution: str = "monthly", start: str = None, end: str = None, path: str = "", @@ -56,12 +56,8 @@ def __init__( lat_lim=lat_lim, lon_lim=lon_lim, fmt=fmt, + path=path, ) - self.output_folder = Path(path).absolute() - - # make directory if it not exists - if not os.path.exists(self.output_folder): - os.makedirs(self.output_folder) def initialize(self, bucket: str = "era5-pds") -> object: """initialize connection with amazon s3 and create a client. @@ -116,7 +112,7 @@ def check_input_dates( elif temporal_resolution == "monthly": self.dates = pd.date_range(self.start, self.end, freq="MS") - def download(self): + def download(self, progress_bar: bool = True): """Download wrapper over all given variables. ECMWF method downloads ECMWF daily data for a given variable, temporal_resolution @@ -138,7 +134,7 @@ def download(self): for var in self.vars: var_info = catalog.get_variable(var) var_s3_name = var_info.get("bucket_name") - self.downloadDataset(var_s3_name) + self.downloadDataset(var_s3_name, progress_bar=progress_bar) def downloadDataset(self, var: str, progress_bar: bool = True): """Download a climate variable. @@ -175,7 +171,7 @@ def downloadDataset(self, var: str, progress_bar: bool = True): month = date.strftime("%m") # file path patterns for remote S3 objects and corresponding local file s3_data_key = f"{year}/{month}/data/{var}.nc" - downloaded_file_dir = f"{self.output_folder}/{year}{month}_{self.temporal_resolution}_{var}.nc" + downloaded_file_dir = f"{self.path}/{year}{month}_{self.temporal_resolution}_{var}.nc" self.API(s3_data_key, downloaded_file_dir) diff --git a/examples/abstract_implementation.py b/examples/abstract_implementation.py index b260695..061966f 100644 --- a/examples/abstract_implementation.py +++ b/examples/abstract_implementation.py @@ -1,11 +1,12 @@ from earth2observe.earth2observe import Earth2Observe - -source = "chirps" +# unified parameters for all data sources. start = "2009-01-01" end = "2009-01-10" temporal_resolution = "daily" latlim = [4.19, 4.64] lonlim = [-75.65, -74.73] +#%% +source = "chirps" path = r"examples\data\chirps" variables = ["precipitation"] e2o = Earth2Observe( @@ -37,7 +38,23 @@ path = r"examples\data\ecmwf" source = "ecmwf" -variables = ["E"] +variables = ["precipitation"] +e2o = Earth2Observe( + data_source=source, + start=start, + end=end, + variables=variables, + lat_lim=latlim, + lon_lim=lonlim, + temporal_resolution=temporal_resolution, + path=path, +) +# e2o.download() + +#%% +path = r"examples\data\s3-backend" +source = "amazon-s3" +variables = ["precipitation"] e2o = Earth2Observe( data_source=source, start=start, diff --git a/examples/chirps_data.py b/examples/chirps_data.py index c046945..84dc751 100644 --- a/examples/chirps_data.py +++ b/examples/chirps_data.py @@ -5,12 +5,12 @@ print(chirps_catalog.catalog) # %% precipitation start = "2009-01-01" -end = "2009-01-10" +end = "2009-01-2" time = "daily" latlim = [4.19, 4.64] lonlim = [-75.65, -74.73] -path = r"examples\data\chirps" +path = r"examples/data/delete/chirps" Coello = CHIRPS( start=start, end=end, diff --git a/examples/ecmwf_data.py b/examples/ecmwf_data.py index 0a246d8..fb16031 100644 --- a/examples/ecmwf_data.py +++ b/examples/ecmwf_data.py @@ -9,7 +9,7 @@ from earth2observe.ecmwf import ECMWF, Catalog rpath = os.getcwd() -path = rf"{rpath}\examples\data\ecmwf" +path = rf"{rpath}\delete\data\ecmwf" #%% precipitation start = "2009-01-01" end = "2009-01-10" diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py index e4178ec..6b46a0c 100644 --- a/examples/s3_er5_data.py +++ b/examples/s3_er5_data.py @@ -18,7 +18,7 @@ lon = [-75.649243, -74.727286] variables = ["precipitation"] rpath = os.getcwd() -path = rf"{rpath}/examples/data/s3/era5" +path = rf"{rpath}/examples/data/delete/s3/era5" s3_era5 = S3( temporal_resolution=time, diff --git a/tests/conftest.py b/tests/conftest.py index 26b5b3c..e68ed42 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,18 +8,27 @@ @pytest.fixture(scope="session") def dates() -> List: - return ["2009-01-01", "2009-01-05"] + return ["2009-01-01", "2009-01-02"] + + +@pytest.fixture(scope="session") +def monthly_dates() -> List: + return ["2009-01-01", "2009-02-01"] @pytest.fixture(scope="session") def number_downloaded_files() -> int: - return 5 + return 2 @pytest.fixture(scope="session") def daily_temporal_resolution() -> str: return "daily" +@pytest.fixture(scope="session") +def monthly_temporal_resolution() -> str: + return "monthly" + @pytest.fixture(scope="session") def lat_bounds() -> List: @@ -40,32 +49,45 @@ def chirps_base_dir() -> str: def ecmwf_base_dir() -> Path: return Path("tests/data/delete/ecmwf").absolute() +@pytest.fixture(scope="session") +def s3_era5_base_dir() -> Path: + return Path("tests/data/delete/s3/era5").absolute() @pytest.fixture(scope="session") def chirps_variables() -> List[str]: return ["precipitation"] # "T", - @pytest.fixture(scope="session") def ecmwf_variables() -> List[str]: return ["E"] # "T", +@pytest.fixture(scope="session") +def s3_era5_variables() -> List[str]: + return ["precipitation"] + @pytest.fixture(scope="session") def ecmwf_data_source() -> str: return "ecmwf" +@pytest.fixture(scope="session") +def chirps_data_source() -> str: + return "chirps" @pytest.fixture(scope="session") -def ecmwf_data_source_output_dir() -> str: - return Path("tests/data/delete/ecmwf-backend").absolute() +def s3_data_source() -> str: + return "amazon-s3" @pytest.fixture(scope="session") -def chirps_data_source() -> str: - return "chirps" +def ecmwf_data_source_output_dir() -> str: + return Path("tests/data/delete/ecmwf-backend").absolute() @pytest.fixture(scope="session") def chirps_data_source_output_dir() -> str: return Path("tests/data/delete/chirps-backend").absolute() + +@pytest.fixture(scope="session") +def s3_era5_data_source_output_dir() -> str: + return Path("tests/data/delete/s3-era5").absolute() From ada6289ab86f0784c08126b4425470f5a3199e56 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 22:53:14 +0100 Subject: [PATCH 14/33] update tests --- tests/test_earth2observe.py | 56 +++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 6651cd9..21a540c 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -8,6 +8,7 @@ from earth2observe.chirps import CHIRPS from earth2observe.earth2observe import Earth2Observe from earth2observe.ecmwf import ECMWF +from earth2observe.s3 import S3 class TestChirpsBackend: @@ -53,7 +54,10 @@ def test_download_chirps_backend( ) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{chirps_data_source_output_dir}/chirps/precipitation") + try: + shutil.rmtree(f"{chirps_data_source_output_dir}/chirps/precipitation") + except PermissionError: + print("the downloaded files could not be deleted") class TestECMWFBackend: @@ -95,4 +99,52 @@ def test_download_ecmwf_backend( ) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{ecmwf_data_source_output_dir}/daily") + try: + shutil.rmtree(f"{ecmwf_data_source_output_dir}/daily") + except PermissionError: + print("the downloaded files could not be deleted") + + +class TestS3Backend: + @pytest.fixture(scope="session") + def test_s3_data_source_instantiate_object( + self, + s3_data_source: str, + monthly_dates: List, + monthly_temporal_resolution: str, + s3_era5_variables: List[str], + lat_bounds: List, + lon_bounds: List, + s3_era5_data_source_output_dir: str, + ): + e2o = Earth2Observe( + data_source=s3_data_source, + start=monthly_dates[0], + end=monthly_dates[1], + variables=s3_era5_variables, + lat_lim=lat_bounds, + lon_lim=lon_bounds, + temporal_resolution=monthly_temporal_resolution, + path=s3_era5_data_source_output_dir, + ) + assert isinstance(e2o.DataSources, dict) + assert isinstance(e2o.datasource, S3) + assert e2o.datasource.vars == s3_era5_variables + return e2o + + def test_download_ecmwf_backend( + self, + test_s3_data_source_instantiate_object: S3, + s3_era5_data_source_output_dir: str, + number_downloaded_files: int, + ): + test_s3_data_source_instantiate_object.download() + filelist = glob.glob( + os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") + ) + assert len(filelist) == number_downloaded_files + # delete the files + try: + shutil.rmtree(f"{s3_era5_data_source_output_dir}") + except PermissionError: + print("the downloaded files could not be deleted") From c593d10c79142f459d9545226b4898f43a160f50 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 23:09:02 +0100 Subject: [PATCH 15/33] update tests --- tests/test_chirps.py | 5 ++++- tests/test_earth2observe.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_chirps.py b/tests/test_chirps.py index 61b684d..b641f7c 100644 --- a/tests/test_chirps.py +++ b/tests/test_chirps.py @@ -48,4 +48,7 @@ def test_download( ) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{chirps_base_dir}") + try: + shutil.rmtree(f"{chirps_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted") diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 21a540c..81c29cf 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -49,13 +49,13 @@ def test_download_chirps_backend( fname = "P_CHIRPS" filelist = glob.glob( os.path.join( - f"{chirps_data_source_output_dir}/chirps/precipitation", f"{fname}*.tif" + f"{chirps_data_source_output_dir}", f"{fname}*.tif" ) ) assert len(filelist) == number_downloaded_files # delete the files try: - shutil.rmtree(f"{chirps_data_source_output_dir}/chirps/precipitation") + shutil.rmtree(f"{chirps_data_source_output_dir}") except PermissionError: print("the downloaded files could not be deleted") @@ -95,12 +95,12 @@ def test_download_ecmwf_backend( ): test_ecmwf_data_source_instantiate_object.download() filelist = glob.glob( - os.path.join(f"{ecmwf_data_source_output_dir}/daily/Evaporation/", f"*.tif") + os.path.join(f"{ecmwf_data_source_output_dir}", f"*.tif") ) assert len(filelist) == number_downloaded_files # delete the files try: - shutil.rmtree(f"{ecmwf_data_source_output_dir}/daily") + shutil.rmtree(f"{ecmwf_data_source_output_dir}") except PermissionError: print("the downloaded files could not be deleted") From 1815a72fea4a968051a2310c5056dfd86fb1b907 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Mon, 16 Jan 2023 23:22:25 +0100 Subject: [PATCH 16/33] update paths --- tests/conftest.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e68ed42..2dd9dbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ +import os from pathlib import Path -from typing import List - import pytest from tests.gee.conftest import * @@ -42,16 +41,28 @@ def lon_bounds() -> List: @pytest.fixture(scope="session") def chirps_base_dir() -> str: - return "tests/data/delete/chirps" + path = "tests/data/delete/chirps" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() @pytest.fixture(scope="session") def ecmwf_base_dir() -> Path: - return Path("tests/data/delete/ecmwf").absolute() + path = "tests/data/delete/ecmwf" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() @pytest.fixture(scope="session") def s3_era5_base_dir() -> Path: - return Path("tests/data/delete/s3/era5").absolute() + path = "tests/data/delete/s3/era5" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() @pytest.fixture(scope="session") def chirps_variables() -> List[str]: @@ -81,13 +92,25 @@ def s3_data_source() -> str: @pytest.fixture(scope="session") def ecmwf_data_source_output_dir() -> str: - return Path("tests/data/delete/ecmwf-backend").absolute() + path = "tests/data/delete/ecmwf-backend" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() @pytest.fixture(scope="session") def chirps_data_source_output_dir() -> str: - return Path("tests/data/delete/chirps-backend").absolute() + path = "tests/data/delete/chirps-backend" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() @pytest.fixture(scope="session") def s3_era5_data_source_output_dir() -> str: - return Path("tests/data/delete/s3-era5").absolute() + path = "tests/data/delete/s3-era5" + return path + # if not os.path.exists(path): + # os.mkdir(path) + # return Path(path).absolute() From ef17a62a26508665d2c096ff7643287864d898d9 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 00:26:03 +0100 Subject: [PATCH 17/33] update tests --- tests/conftest.py | 57 ++++++++++++++++++------------------- tests/test_earth2observe.py | 2 +- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2dd9dbe..6a9c0e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os -from pathlib import Path -import pytest +import pytest +from pathlib import Path from tests.gee.conftest import * @@ -42,27 +42,27 @@ def lon_bounds() -> List: @pytest.fixture(scope="session") def chirps_base_dir() -> str: path = "tests/data/delete/chirps" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() @pytest.fixture(scope="session") -def ecmwf_base_dir() -> Path: +def ecmwf_base_dir() -> str: path = "tests/data/delete/ecmwf" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + @pytest.fixture(scope="session") -def s3_era5_base_dir() -> Path: +def s3_era5_base_dir() -> str: path = "tests/data/delete/s3/era5" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + + @pytest.fixture(scope="session") def chirps_variables() -> List[str]: @@ -93,24 +93,23 @@ def s3_data_source() -> str: @pytest.fixture(scope="session") def ecmwf_data_source_output_dir() -> str: path = "tests/data/delete/ecmwf-backend" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() @pytest.fixture(scope="session") def chirps_data_source_output_dir() -> str: path = "tests/data/delete/chirps-backend" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + @pytest.fixture(scope="session") def s3_era5_data_source_output_dir() -> str: - path = "tests/data/delete/s3-era5" - return path - # if not os.path.exists(path): - # os.mkdir(path) - # return Path(path).absolute() + path = "tests/data/delete/s3-era5000" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 81c29cf..ead0877 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -132,7 +132,7 @@ def test_s3_data_source_instantiate_object( assert e2o.datasource.vars == s3_era5_variables return e2o - def test_download_ecmwf_backend( + def test_download_s3_backend( self, test_s3_data_source_instantiate_object: S3, s3_era5_data_source_output_dir: str, From eefa6791797755c367a9d3877d0b759f7fea8599 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 00:48:47 +0100 Subject: [PATCH 18/33] change to module fixtures --- tests/conftest.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6a9c0e8..2d86f11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,25 +29,25 @@ def monthly_temporal_resolution() -> str: return "monthly" -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def lat_bounds() -> List: return [4.19, 4.64] -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def lon_bounds() -> List: return [-75.65, -74.73] -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def chirps_base_dir() -> str: - path = "tests/data/delete/chirps" - if not os.path.exists(path): - os.makedirs(path) - return Path(path).absolute() + rpath = Path(f"tests/data/delete/chirps") + if not os.path.exists(rpath): + os.makedirs(rpath) + return Path(rpath).absolute() -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def ecmwf_base_dir() -> str: path = "tests/data/delete/ecmwf" if not os.path.exists(path): @@ -55,15 +55,13 @@ def ecmwf_base_dir() -> str: return Path(path).absolute() -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def s3_era5_base_dir() -> str: path = "tests/data/delete/s3/era5" if not os.path.exists(path): os.makedirs(path) return Path(path).absolute() - - @pytest.fixture(scope="session") def chirps_variables() -> List[str]: return ["precipitation"] # "T", @@ -90,7 +88,7 @@ def s3_data_source() -> str: return "amazon-s3" -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def ecmwf_data_source_output_dir() -> str: path = "tests/data/delete/ecmwf-backend" if not os.path.exists(path): @@ -98,7 +96,7 @@ def ecmwf_data_source_output_dir() -> str: return Path(path).absolute() -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def chirps_data_source_output_dir() -> str: path = "tests/data/delete/chirps-backend" if not os.path.exists(path): @@ -106,7 +104,7 @@ def chirps_data_source_output_dir() -> str: return Path(path).absolute() -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def s3_era5_data_source_output_dir() -> str: path = "tests/data/delete/s3-era5000" if not os.path.exists(path): From 15b47da0c1b4b0ddb13b1803a59cf78aa02f8c9a Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 00:50:22 +0100 Subject: [PATCH 19/33] change to module fixtures --- tests/test_chirps.py | 2 +- tests/test_earth2observe.py | 6 +++--- tests/test_ecmwf.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_chirps.py b/tests/test_chirps.py index b641f7c..2a13c84 100644 --- a/tests/test_chirps.py +++ b/tests/test_chirps.py @@ -8,7 +8,7 @@ from earth2observe.chirps import CHIRPS -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def test_create_chirps_object( dates: List, daily_temporal_resolution: str, diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index ead0877..607821f 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -12,7 +12,7 @@ class TestChirpsBackend: - @pytest.fixture(scope="session") + @pytest.fixture(scope="module") def test_chirps_data_source_instantiate_object( self, chirps_data_source: str, @@ -61,7 +61,7 @@ def test_download_chirps_backend( class TestECMWFBackend: - @pytest.fixture(scope="session") + @pytest.fixture(scope="module") def test_ecmwf_data_source_instantiate_object( self, ecmwf_data_source: str, @@ -106,7 +106,7 @@ def test_download_ecmwf_backend( class TestS3Backend: - @pytest.fixture(scope="session") + @pytest.fixture(scope="module") def test_s3_data_source_instantiate_object( self, s3_data_source: str, diff --git a/tests/test_ecmwf.py b/tests/test_ecmwf.py index d7601ed..d35af2c 100644 --- a/tests/test_ecmwf.py +++ b/tests/test_ecmwf.py @@ -8,7 +8,7 @@ from earth2observe.ecmwf import ECMWF -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def test_create_ecmwf_object( dates: List, lat_bounds: List, From e8d8776a2c18016a43ebe3cc25318774f21bcf3f Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 20:00:21 +0100 Subject: [PATCH 20/33] comment s3 tests because of an error in ci --- tests/test_earth2observe.py | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 607821f..5d3738c 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -105,46 +105,46 @@ def test_download_ecmwf_backend( print("the downloaded files could not be deleted") -class TestS3Backend: - @pytest.fixture(scope="module") - def test_s3_data_source_instantiate_object( - self, - s3_data_source: str, - monthly_dates: List, - monthly_temporal_resolution: str, - s3_era5_variables: List[str], - lat_bounds: List, - lon_bounds: List, - s3_era5_data_source_output_dir: str, - ): - e2o = Earth2Observe( - data_source=s3_data_source, - start=monthly_dates[0], - end=monthly_dates[1], - variables=s3_era5_variables, - lat_lim=lat_bounds, - lon_lim=lon_bounds, - temporal_resolution=monthly_temporal_resolution, - path=s3_era5_data_source_output_dir, - ) - assert isinstance(e2o.DataSources, dict) - assert isinstance(e2o.datasource, S3) - assert e2o.datasource.vars == s3_era5_variables - return e2o - - def test_download_s3_backend( - self, - test_s3_data_source_instantiate_object: S3, - s3_era5_data_source_output_dir: str, - number_downloaded_files: int, - ): - test_s3_data_source_instantiate_object.download() - filelist = glob.glob( - os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") - ) - assert len(filelist) == number_downloaded_files - # delete the files - try: - shutil.rmtree(f"{s3_era5_data_source_output_dir}") - except PermissionError: - print("the downloaded files could not be deleted") +# class TestS3Backend: +# @pytest.fixture(scope="module") +# def test_s3_data_source_instantiate_object( +# self, +# s3_data_source: str, +# monthly_dates: List, +# monthly_temporal_resolution: str, +# s3_era5_variables: List[str], +# lat_bounds: List, +# lon_bounds: List, +# s3_era5_data_source_output_dir: str, +# ): +# e2o = Earth2Observe( +# data_source=s3_data_source, +# start=monthly_dates[0], +# end=monthly_dates[1], +# variables=s3_era5_variables, +# lat_lim=lat_bounds, +# lon_lim=lon_bounds, +# temporal_resolution=monthly_temporal_resolution, +# path=s3_era5_data_source_output_dir, +# ) +# assert isinstance(e2o.DataSources, dict) +# assert isinstance(e2o.datasource, S3) +# assert e2o.datasource.vars == s3_era5_variables +# return e2o +# +# def test_download_s3_backend( +# self, +# test_s3_data_source_instantiate_object: S3, +# s3_era5_data_source_output_dir: str, +# number_downloaded_files: int, +# ): +# test_s3_data_source_instantiate_object.download() +# filelist = glob.glob( +# os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") +# ) +# assert len(filelist) == number_downloaded_files +# # delete the files +# try: +# shutil.rmtree(f"{s3_era5_data_source_output_dir}") +# except PermissionError: +# print("the downloaded files could not be deleted") From 3c8f81e51522a8c7f1938f6f8f6e538a523021ab Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 20:07:44 +0100 Subject: [PATCH 21/33] add test dwnload folders --- tests/data/delete/chirps-backend/.gitignore | 7 +++++++ tests/data/delete/chirps/.gitignore | 7 +++++++ tests/data/delete/ecmwf-backend/.gitignore | 7 +++++++ tests/data/delete/ecmwf/.gitignore | 7 +++++++ tests/data/delete/s3-era5-backend/.gitignore | 7 +++++++ tests/data/delete/s3-era5/.gitignore | 7 +++++++ 6 files changed, 42 insertions(+) create mode 100644 tests/data/delete/chirps-backend/.gitignore create mode 100644 tests/data/delete/chirps/.gitignore create mode 100644 tests/data/delete/ecmwf-backend/.gitignore create mode 100644 tests/data/delete/ecmwf/.gitignore create mode 100644 tests/data/delete/s3-era5-backend/.gitignore create mode 100644 tests/data/delete/s3-era5/.gitignore diff --git a/tests/data/delete/chirps-backend/.gitignore b/tests/data/delete/chirps-backend/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/chirps-backend/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/chirps/.gitignore b/tests/data/delete/chirps/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/chirps/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/ecmwf-backend/.gitignore b/tests/data/delete/ecmwf-backend/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/ecmwf-backend/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/ecmwf/.gitignore b/tests/data/delete/ecmwf/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/ecmwf/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/s3-era5-backend/.gitignore b/tests/data/delete/s3-era5-backend/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/s3-era5-backend/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/tests/data/delete/s3-era5/.gitignore b/tests/data/delete/s3-era5/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/tests/data/delete/s3-era5/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore From 4ada7b83d16953bea72ae1801f6ddab29cc11368 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 20:08:22 +0100 Subject: [PATCH 22/33] change download folder names --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2d86f11..daf1972 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,7 +57,7 @@ def ecmwf_base_dir() -> str: @pytest.fixture(scope="module") def s3_era5_base_dir() -> str: - path = "tests/data/delete/s3/era5" + path = "tests/data/delete/s3-era5" if not os.path.exists(path): os.makedirs(path) return Path(path).absolute() @@ -106,7 +106,7 @@ def chirps_data_source_output_dir() -> str: @pytest.fixture(scope="module") def s3_era5_data_source_output_dir() -> str: - path = "tests/data/delete/s3-era5000" + path = "tests/data/delete/s3-era5-backend" if not os.path.exists(path): os.makedirs(path) return Path(path).absolute() From 7f515490fb2af939cc538a703cad22e848081d5c Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 20:42:43 +0100 Subject: [PATCH 23/33] fix directory issues --- earth2observe/ecmwf.py | 7 +- tests/data/delete/chirps-backend/.gitignore | 7 -- tests/data/delete/chirps/.gitignore | 7 -- tests/data/delete/ecmwf-backend/.gitignore | 7 -- tests/data/delete/ecmwf/.gitignore | 7 -- tests/data/delete/s3-era5-backend/.gitignore | 7 -- tests/data/delete/s3-era5/.gitignore | 7 -- tests/test_earth2observe.py | 86 ++++++++++---------- 8 files changed, 45 insertions(+), 90 deletions(-) delete mode 100644 tests/data/delete/chirps-backend/.gitignore delete mode 100644 tests/data/delete/chirps/.gitignore delete mode 100644 tests/data/delete/ecmwf-backend/.gitignore delete mode 100644 tests/data/delete/ecmwf/.gitignore delete mode 100644 tests/data/delete/s3-era5-backend/.gitignore delete mode 100644 tests/data/delete/s3-era5/.gitignore diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index c6b0778..f14470d 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -322,9 +322,6 @@ def callAPI( area_str dataset """ - - os.chdir(output_folder) - if download_type == 1 or download_type == 2: server.retrieve( { @@ -340,7 +337,7 @@ def callAPI( "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": area_str, "format": "netcdf", - "target": f"data_{dataset}.nc", + "target": f"{output_folder}/data_{dataset}.nc", } ) @@ -360,7 +357,7 @@ def callAPI( "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": area_str, "format": "netcdf", - "target": f"data_{dataset}.nc", + "target": f"{output_folder}/data_{dataset}.nc", } ) diff --git a/tests/data/delete/chirps-backend/.gitignore b/tests/data/delete/chirps-backend/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/chirps-backend/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/data/delete/chirps/.gitignore b/tests/data/delete/chirps/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/chirps/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/data/delete/ecmwf-backend/.gitignore b/tests/data/delete/ecmwf-backend/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/ecmwf-backend/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/data/delete/ecmwf/.gitignore b/tests/data/delete/ecmwf/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/ecmwf/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/data/delete/s3-era5-backend/.gitignore b/tests/data/delete/s3-era5-backend/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/s3-era5-backend/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/data/delete/s3-era5/.gitignore b/tests/data/delete/s3-era5/.gitignore deleted file mode 100644 index f2c3201..0000000 --- a/tests/data/delete/s3-era5/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -# Ignore everything in this directory - -* - -# Except this file - -!.gitignore diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 5d3738c..607821f 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -105,46 +105,46 @@ def test_download_ecmwf_backend( print("the downloaded files could not be deleted") -# class TestS3Backend: -# @pytest.fixture(scope="module") -# def test_s3_data_source_instantiate_object( -# self, -# s3_data_source: str, -# monthly_dates: List, -# monthly_temporal_resolution: str, -# s3_era5_variables: List[str], -# lat_bounds: List, -# lon_bounds: List, -# s3_era5_data_source_output_dir: str, -# ): -# e2o = Earth2Observe( -# data_source=s3_data_source, -# start=monthly_dates[0], -# end=monthly_dates[1], -# variables=s3_era5_variables, -# lat_lim=lat_bounds, -# lon_lim=lon_bounds, -# temporal_resolution=monthly_temporal_resolution, -# path=s3_era5_data_source_output_dir, -# ) -# assert isinstance(e2o.DataSources, dict) -# assert isinstance(e2o.datasource, S3) -# assert e2o.datasource.vars == s3_era5_variables -# return e2o -# -# def test_download_s3_backend( -# self, -# test_s3_data_source_instantiate_object: S3, -# s3_era5_data_source_output_dir: str, -# number_downloaded_files: int, -# ): -# test_s3_data_source_instantiate_object.download() -# filelist = glob.glob( -# os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") -# ) -# assert len(filelist) == number_downloaded_files -# # delete the files -# try: -# shutil.rmtree(f"{s3_era5_data_source_output_dir}") -# except PermissionError: -# print("the downloaded files could not be deleted") +class TestS3Backend: + @pytest.fixture(scope="module") + def test_s3_data_source_instantiate_object( + self, + s3_data_source: str, + monthly_dates: List, + monthly_temporal_resolution: str, + s3_era5_variables: List[str], + lat_bounds: List, + lon_bounds: List, + s3_era5_data_source_output_dir: str, + ): + e2o = Earth2Observe( + data_source=s3_data_source, + start=monthly_dates[0], + end=monthly_dates[1], + variables=s3_era5_variables, + lat_lim=lat_bounds, + lon_lim=lon_bounds, + temporal_resolution=monthly_temporal_resolution, + path=s3_era5_data_source_output_dir, + ) + assert isinstance(e2o.DataSources, dict) + assert isinstance(e2o.datasource, S3) + assert e2o.datasource.vars == s3_era5_variables + return e2o + + def test_download_s3_backend( + self, + test_s3_data_source_instantiate_object: S3, + s3_era5_data_source_output_dir: str, + number_downloaded_files: int, + ): + test_s3_data_source_instantiate_object.download() + filelist = glob.glob( + os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") + ) + assert len(filelist) == number_downloaded_files + # delete the files + try: + shutil.rmtree(f"{s3_era5_data_source_output_dir}") + except PermissionError: + print("the downloaded files could not be deleted") From b689565f79c58b05ace3e0dcf9078467df1bacb9 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Tue, 17 Jan 2023 20:43:39 +0100 Subject: [PATCH 24/33] update tests --- .gitignore | 2 +- tests/test_s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 39cd94c..48948a7 100644 --- a/.gitignore +++ b/.gitignore @@ -146,7 +146,7 @@ build_artifacts .idea mo_* conda/ -/examples/data/ecmwf/daily/* +/examples/data/ecmwf/* .idea/* examples/data/chirps/* *.xml diff --git a/tests/test_s3.py b/tests/test_s3.py index 8ea9421..e33dab4 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -5,7 +5,7 @@ import pytest from earth2observe.s3 import S3 -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def test_create_s3_object( monthly_dates: List, lat_bounds: List, From dff8c9ec0b9c4c8cdcb0ee195e09e9abaddadd6d Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:03:55 +0100 Subject: [PATCH 25/33] bump up pyramids version --- environment.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index f010b70..b0ce608 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,7 @@ dependencies: - earthengine-api >=0.1.324 - boto3 >=1.26.50 - joblib >=1.2.0 - - pyramids >=0.2.12 + - pyramids >=0.3.1 - serapeum_utils >=0.1.1 - loguru >=0.6.0 - PyYAML >=6.0 diff --git a/requirements.txt b/requirements.txt index a209c72..46c8842 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ numpy ==1.24.1 pandas >=1.4.4 pathlib >=1.0.1 pip >=22.3.1 -pyramids-gis >=0.2.12 +pyramids-gis >=0.3.1 serapeum_utils >=0.1.1 PyYAML >=6.0 requests >=2.28.1 From 5ad663b687f7af0608bea46c16f8634a4ef86252 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:12:17 +0100 Subject: [PATCH 26/33] update examples --- examples/s3_er5_data.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py index 6b46a0c..c31f50b 100644 --- a/examples/s3_er5_data.py +++ b/examples/s3_er5_data.py @@ -1,7 +1,6 @@ import os from earth2observe.s3 import S3, Catalog - #%% s3_catalog = Catalog() print(s3_catalog.catalog) @@ -11,14 +10,14 @@ # available_date_abs_path = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=True) # available_date = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=False) #%% -start = "2009-01-01" -end = "2009-05-01" +start = "2022-05-01" +end = "2022-05-01" time = "monthly" lat = [4.190755, 4.643963] lon = [-75.649243, -74.727286] variables = ["precipitation"] rpath = os.getcwd() -path = rf"{rpath}/examples/data/delete/s3/era5" +path = rf"{rpath}/examples/data/s3-era5" s3_era5 = S3( temporal_resolution=time, @@ -26,8 +25,9 @@ end=end, path=path, variables=variables, - lat_lim=lat, - lon_lim=lon, + # lat_lim=lat, + # lon_lim=lon, ) #%% s3_era5.download() +#%% From 8ebea3a65d16e9da9c585c7b180a9901213e5e97 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:12:43 +0100 Subject: [PATCH 27/33] add default lon and lat limits --- earth2observe/earth2observe.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/earth2observe/earth2observe.py b/earth2observe/earth2observe.py index 841c749..dbf87d1 100644 --- a/earth2observe/earth2observe.py +++ b/earth2observe/earth2observe.py @@ -3,7 +3,8 @@ from earth2observe.ecmwf import ECMWF from earth2observe.s3 import S3 - +DEFAULT_LONGITUDE_LIMIT = [-180, 180] +DEFAULT_LATITUDE_LIMIT = [-90, 90] class Earth2Observe: """End user class to call all the data source classes abailable in earth2observe.""" @@ -23,6 +24,10 @@ def __init__( ): if data_source not in self.DataSources: raise ValueError(f"{data_source} not supported") + if lat_lim is None: + lat_lim = DEFAULT_LATITUDE_LIMIT + if lon_lim is None: + lon_lim = DEFAULT_LONGITUDE_LIMIT self.datasource = self.DataSources[data_source]( start=start, From af1c3455386c5ce40846e7c3008e7f1727d1ac77 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:14:16 +0100 Subject: [PATCH 28/33] update examples --- examples/abstract_implementation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/abstract_implementation.py b/examples/abstract_implementation.py index 061966f..07381d5 100644 --- a/examples/abstract_implementation.py +++ b/examples/abstract_implementation.py @@ -60,8 +60,8 @@ start=start, end=end, variables=variables, - lat_lim=latlim, - lon_lim=lonlim, + # lat_lim=latlim, + # lon_lim=lonlim, temporal_resolution=temporal_resolution, path=path, ) From c2dea0ff61bac13b4c23b09f21692d37754f6ce2 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:24:17 +0100 Subject: [PATCH 29/33] bump up versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2fa0ea2..bce4d80 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name="earth2observe", - version="0.2.0", + version="0.2.1", description="remote sensing package", author="Mostafa Farrag", author_email="moah.farag@gmail.come", From 52ba052bf8ef5c08b68b0966c7e3ff9e3c5db8bb Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:27:55 +0100 Subject: [PATCH 30/33] refactor --- earth2observe/abstractdatasource.py | 4 ++-- earth2observe/chirps.py | 3 +-- earth2observe/earth2observe.py | 2 ++ examples/abstract_implementation.py | 1 + tests/conftest.py | 10 ++++++++-- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index 8749a0d..c635750 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -1,7 +1,7 @@ import os -from typing import Dict -from pathlib import Path from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict class AbstractDataSource(ABC): diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index d0a770d..d7b0dc6 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -1,7 +1,6 @@ import datetime as dt import os from ftplib import FTP -from pathlib import Path import numpy as np import pandas as pd @@ -9,9 +8,9 @@ from osgeo import gdal from pyramids.raster import Raster from pyramids.utils import extractFromGZ +from serapeum_utils.utils import print_progress_bar from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from serapeum_utils.utils import print_progress_bar class CHIRPS(AbstractDataSource): diff --git a/earth2observe/earth2observe.py b/earth2observe/earth2observe.py index dbf87d1..421cbd5 100644 --- a/earth2observe/earth2observe.py +++ b/earth2observe/earth2observe.py @@ -5,6 +5,8 @@ DEFAULT_LONGITUDE_LIMIT = [-180, 180] DEFAULT_LATITUDE_LIMIT = [-90, 90] + + class Earth2Observe: """End user class to call all the data source classes abailable in earth2observe.""" diff --git a/examples/abstract_implementation.py b/examples/abstract_implementation.py index 07381d5..0eda78c 100644 --- a/examples/abstract_implementation.py +++ b/examples/abstract_implementation.py @@ -1,4 +1,5 @@ from earth2observe.earth2observe import Earth2Observe + # unified parameters for all data sources. start = "2009-01-01" end = "2009-01-10" diff --git a/tests/conftest.py b/tests/conftest.py index daf1972..ae9466a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import os +from pathlib import Path import pytest -from pathlib import Path + from tests.gee.conftest import * @@ -24,6 +25,7 @@ def number_downloaded_files() -> int: def daily_temporal_resolution() -> str: return "daily" + @pytest.fixture(scope="session") def monthly_temporal_resolution() -> str: return "monthly" @@ -62,14 +64,17 @@ def s3_era5_base_dir() -> str: os.makedirs(path) return Path(path).absolute() + @pytest.fixture(scope="session") def chirps_variables() -> List[str]: return ["precipitation"] # "T", + @pytest.fixture(scope="session") def ecmwf_variables() -> List[str]: return ["E"] # "T", + @pytest.fixture(scope="session") def s3_era5_variables() -> List[str]: return ["precipitation"] @@ -79,10 +84,12 @@ def s3_era5_variables() -> List[str]: def ecmwf_data_source() -> str: return "ecmwf" + @pytest.fixture(scope="session") def chirps_data_source() -> str: return "chirps" + @pytest.fixture(scope="session") def s3_data_source() -> str: return "amazon-s3" @@ -110,4 +117,3 @@ def s3_era5_data_source_output_dir() -> str: if not os.path.exists(path): os.makedirs(path) return Path(path).absolute() - From fbc087a62bb0f6f9f07d38d78d234f4fc3705c33 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:28:55 +0100 Subject: [PATCH 31/33] refactor --- earth2observe/ecmwf.py | 4 ++-- earth2observe/s3.py | 7 ++++--- examples/s3_er5_data.py | 1 + requirements.txt | 2 +- tests/test_chirps.py | 4 +--- tests/test_earth2observe.py | 12 +++--------- tests/test_ecmwf.py | 1 - tests/test_s3.py | 5 ++++- 8 files changed, 16 insertions(+), 20 deletions(-) diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index f14470d..2850906 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -5,7 +5,6 @@ import calendar import datetime as dt import os -from pathlib import Path from typing import Dict import numpy as np @@ -15,10 +14,11 @@ from loguru import logger from netCDF4 import Dataset from pyramids.raster import Raster +from serapeum_utils.utils import print_progress_bar from earth2observe import __path__ from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from serapeum_utils.utils import print_progress_bar + class ECMWF(AbstractDataSource): """RemoteSensing. diff --git a/earth2observe/s3.py b/earth2observe/s3.py index a91c3be..3c22926 100644 --- a/earth2observe/s3.py +++ b/earth2observe/s3.py @@ -1,16 +1,15 @@ """Amazon S3.""" import datetime as dt import os -from pathlib import Path from typing import Dict, List import boto3 import botocore import pandas as pd from botocore import exceptions +from serapeum_utils.utils import print_progress_bar from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from serapeum_utils.utils import print_progress_bar class S3(AbstractDataSource): @@ -171,7 +170,9 @@ def downloadDataset(self, var: str, progress_bar: bool = True): month = date.strftime("%m") # file path patterns for remote S3 objects and corresponding local file s3_data_key = f"{year}/{month}/data/{var}.nc" - downloaded_file_dir = f"{self.path}/{year}{month}_{self.temporal_resolution}_{var}.nc" + downloaded_file_dir = ( + f"{self.path}/{year}{month}_{self.temporal_resolution}_{var}.nc" + ) self.API(s3_data_key, downloaded_file_dir) diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py index c31f50b..2d9e81f 100644 --- a/examples/s3_er5_data.py +++ b/examples/s3_er5_data.py @@ -1,6 +1,7 @@ import os from earth2observe.s3 import S3, Catalog + #%% s3_catalog = Catalog() print(s3_catalog.catalog) diff --git a/requirements.txt b/requirements.txt index 46c8842..8600d6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ pandas >=1.4.4 pathlib >=1.0.1 pip >=22.3.1 pyramids-gis >=0.3.1 -serapeum_utils >=0.1.1 PyYAML >=6.0 requests >=2.28.1 +serapeum_utils >=0.1.1 diff --git a/tests/test_chirps.py b/tests/test_chirps.py index 2a13c84..5f6f2fe 100644 --- a/tests/test_chirps.py +++ b/tests/test_chirps.py @@ -43,9 +43,7 @@ def test_download( fname = test_create_chirps_object.clipped_fname test_create_chirps_object.download() - filelist = glob.glob( - os.path.join(f"{chirps_base_dir}", f"{fname}*.tif") - ) + filelist = glob.glob(os.path.join(f"{chirps_base_dir}", f"{fname}*.tif")) assert len(filelist) == number_downloaded_files # delete the files try: diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 607821f..fb10850 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -48,9 +48,7 @@ def test_download_chirps_backend( test_chirps_data_source_instantiate_object.download() fname = "P_CHIRPS" filelist = glob.glob( - os.path.join( - f"{chirps_data_source_output_dir}", f"{fname}*.tif" - ) + os.path.join(f"{chirps_data_source_output_dir}", f"{fname}*.tif") ) assert len(filelist) == number_downloaded_files # delete the files @@ -94,9 +92,7 @@ def test_download_ecmwf_backend( number_downloaded_files: int, ): test_ecmwf_data_source_instantiate_object.download() - filelist = glob.glob( - os.path.join(f"{ecmwf_data_source_output_dir}", f"*.tif") - ) + filelist = glob.glob(os.path.join(f"{ecmwf_data_source_output_dir}", f"*.tif")) assert len(filelist) == number_downloaded_files # delete the files try: @@ -139,9 +135,7 @@ def test_download_s3_backend( number_downloaded_files: int, ): test_s3_data_source_instantiate_object.download() - filelist = glob.glob( - os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc") - ) + filelist = glob.glob(os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc")) assert len(filelist) == number_downloaded_files # delete the files try: diff --git a/tests/test_ecmwf.py b/tests/test_ecmwf.py index d35af2c..bbacc66 100644 --- a/tests/test_ecmwf.py +++ b/tests/test_ecmwf.py @@ -41,4 +41,3 @@ def test_download( shutil.rmtree(f"{ecmwf_base_dir}") except PermissionError: print("the downloaded files could not be deleted") - diff --git a/tests/test_s3.py b/tests/test_s3.py index e33dab4..dd3d930 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -1,10 +1,13 @@ -from typing import List import glob import os import shutil +from typing import List + import pytest + from earth2observe.s3 import S3 + @pytest.fixture(scope="module") def test_create_s3_object( monthly_dates: List, From 14b33c8705b9cd8093a2b668dfaa6f5d2754d7d5 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:29:13 +0100 Subject: [PATCH 32/33] refactor --- earth2observe/gee/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earth2observe/gee/dataset.py b/earth2observe/gee/dataset.py index 0d021fa..67f7847 100644 --- a/earth2observe/gee/dataset.py +++ b/earth2observe/gee/dataset.py @@ -59,10 +59,10 @@ def getDate( Returns ------- start_date: [str] - beginning of the time series. + beginning of the temporal_resolution series. end_date: [str] - end of the time series. + end of the temporal_resolution series. """ data = catalog.loc[catalog["dataset"] == dataset_id, :] From 7bd7bdb8251a0562e398ae2a8d1fffb45ca8e80e Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 25 Jan 2023 01:33:37 +0100 Subject: [PATCH 33/33] update features section --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 03e7a4f..8a8fb42 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Main Features ------------- - ERA Interim Download - CHIRPS Rainfall data Download + - ERA5 from Amason S3 data source Future work @@ -61,7 +62,7 @@ pip install git+https://github.com/MAfarrag/earthobserve ## pip to install the last release you can easly use pip ``` -pip install earthobserve==0.2.0 +pip install earthobserve==0.2.1 ``` Quick start