From 778c1f7fad85aa211f1123c15e2a52f13a9bff31 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Sun, 15 Jan 2023 22:10:57 +0100 Subject: [PATCH] refactor --- earth2observe/abstractdatasource.py | 41 +++++----- earth2observe/chirps.py | 86 +++++++++++--------- earth2observe/earth2observe.py | 44 ++++------- earth2observe/ecmwf.py | 117 ++++++++++++++++------------ 4 files changed, 153 insertions(+), 135 deletions(-) diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index c501420..f28ebea 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -1,21 +1,20 @@ -from typing import Dict from abc import ABC, abstractmethod +from typing import Dict class AbstractDataSource(ABC): - """ - Bluebrint for all class for different datasources - """ + """Bluebrint for all class for different datasources.""" + def __init__( - self, - start: str = None, - end: str = None, - variables: list = None, - temporal_resolution: str = "daily", - lat_lim: list = None, - lon_lim: list = None, - fmt: str = "%Y-%m-%d", - # path: str = "", + self, + start: str = None, + end: str = None, + variables: list = None, + temporal_resolution: str = "daily", + lat_lim: list = None, + lon_lim: list = None, + fmt: str = "%Y-%m-%d", + # path: str = "", ): """ @@ -50,7 +49,7 @@ def __init__( @abstractmethod def check_input_dates(self): - """Check validity of input dates""" + """Check validity of input dates.""" pass @abstractmethod @@ -60,7 +59,7 @@ def initialize(self): @abstractmethod def create_grid(self): - """create a grid from the lat/lon boundaries""" + """create a grid from the lat/lon boundaries.""" pass @abstractmethod @@ -70,29 +69,27 @@ def download(self): # list of dates pass - # @abstractmethod def downloadDataset(self): - """Download single variable/dataset""" + """Download single variable/dataset.""" # used for non ftp servers pass @abstractmethod def API(self): - """send/recieve request to the dataset server""" + """send/recieve request to the dataset server.""" pass - class AbstractCatalog(ABC): - """abstrach class for the datasource catalog""" + """abstrach class for the datasource catalog.""" @abstractmethod def get_catalog(self): - """read the catalog of the datasource from disk or retrieve it from server""" + """read the catalog of the datasource from disk or retrieve it from server.""" pass @abstractmethod def get_variable(self, var_name) -> Dict[str, str]: - """get the details of a specific variable""" + """get the details of a specific variable.""" pass diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index 75e4e80..29b554e 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -1,8 +1,7 @@ -from typing import List import datetime as dt import os -from pathlib import Path from ftplib import FTP +from pathlib import Path import numpy as np import pandas as pd @@ -10,12 +9,14 @@ from osgeo import gdal from pyramids.raster import Raster from pyramids.utils import extractFromGZ -from earth2observe.abstractdatasource import AbstractDataSource, AbstractCatalog + +from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource from earth2observe.utils import print_progress_bar class CHIRPS(AbstractDataSource): """CHIRPS.""" + api_url: str = "data.chc.ucsb.edu" start_date: str = "1981-01-01" end_date: str = "Now" @@ -26,15 +27,15 @@ class CHIRPS(AbstractDataSource): clipped_fname = "P_CHIRPS.v2.0" def __init__( - self, - temporal_resolution: str = "daily", - start: str = None, - end: str = None, - path: str = "", - variables: list = None, - lat_lim: list = None, - lon_lim: list = None, - fmt: str = "%Y-%m-%d", + self, + temporal_resolution: str = "daily", + start: str = None, + end: str = None, + path: str = "", + variables: list = None, + lat_lim: list = None, + lon_lim: list = None, + fmt: str = "%Y-%m-%d", ): """CHIRPS. @@ -66,16 +67,18 @@ def __init__( lon_lim=lon_lim, fmt=fmt, ) - self.output_folder = os.path.join(Path(path).absolute(), "chirps", "precipitation") + self.output_folder = os.path.join( + Path(path).absolute(), "chirps", "precipitation" + ) # make directory if it not exists if not os.path.exists(self.output_folder): os.makedirs(self.output_folder) - - - def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: str): - """check validity of input dates + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): + """check validity of input dates. Parameters ---------- @@ -115,12 +118,12 @@ def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: self.dates = pd.date_range(self.start, self.end, freq=self.time_freq) def initialize(self): - """Initialize FTP server""" + """Initialize FTP server.""" print("FTP server datasources does not need server initialization") pass def create_grid(self, lat_lim: list, lon_lim: list): - """Create_grid + """Create_grid. create grid from the lat/lon boundaries @@ -166,7 +169,6 @@ def create_grid(self, lat_lim: list, lon_lim: list): ) ) - def download(self, progress_bar: bool = True, cores=None, *args, **kwargs): """Download. @@ -228,14 +230,13 @@ def download(self, progress_bar: bool = True, cores=None, *args, **kwargs): return results def API(self, date, args): - """form the request url abd trigger the request + """form the request url abd trigger the request. Parameters ---------- date: args: [list] - """ [output_folder, temp_resolution, xID, yID, lon_lim, latlim] = args @@ -252,28 +253,31 @@ def API(self, date, args): filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif.gz" outfilename = os.path.join( output_folder, - f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif" + f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) DirFileEnd = os.path.join( output_folder, - f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif" + f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) elif temp_resolution == "monthly": - filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz" + filename = ( + f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz" + ) outfilename = os.path.join( output_folder, - f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif" + f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif", ) DirFileEnd = os.path.join( output_folder, - f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif" + f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) else: raise KeyError("The input temporal_resolution interval is not supported") self.callAPI(pathFTP, output_folder, filename) - self.post_download(output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd) - + self.post_download( + output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd + ) @staticmethod def callAPI(pathFTP: str, output_folder: str, filename: str): @@ -314,9 +318,18 @@ def callAPI(pathFTP: str, output_folder: str, filename: str): ftp.retrbinary("RETR " + filename, lf.write, 8192) lf.close() - - def post_download(self, output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd): - """clip the downloaded data to the extent we want + def post_download( + self, + output_folder, + filename, + lon_lim, + latlim, + xID, + yID, + outfilename, + DirFileEnd, + ): + """clip the downloaded data to the extent we want. Parameters ---------- @@ -379,14 +392,15 @@ def listAttributes(self): print("\n") + class Catalog(AbstractCatalog): - """ CHIRPS data catalog""" + """CHIRPS data catalog.""" def __init__(self): self.catalog = self.get_catalog() def get_catalog(self): - """return the catalog""" + """return the catalog.""" return { "Precipitation": { "descriptions": "rainfall [mm/temporal_resolution step]", @@ -394,9 +408,9 @@ def get_catalog(self): "temporal resolution": ["daily", "monthly"], "file name": "rainfall", "var_name": "R", - } + } } def get_variable(self, var_name): - """get the details of a specific variable""" + """get the details of a specific variable.""" return self.catalog.get(var_name) diff --git a/earth2observe/earth2observe.py b/earth2observe/earth2observe.py index 9198000..6ed6ab4 100644 --- a/earth2observe/earth2observe.py +++ b/earth2observe/earth2observe.py @@ -1,31 +1,27 @@ -from earth2observe.ecmwf import ECMWF +"""Front end module that runs each data source backend.""" from earth2observe.chirps import CHIRPS +from earth2observe.ecmwf import ECMWF -class Earth2Observe: - """ - End user class to call all the data source classes abailable in earth2observe. - """ - DataSources = { - "ecmwf": ECMWF, - "chirps": CHIRPS - } +class Earth2Observe: + """End user class to call all the data source classes abailable in earth2observe.""" + DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS} def __init__( - self, - data_source: str = "chirps", - temporal_resolution: str = "daily", - start: str = None, - end: str = None, - path: str = "", - variables: list = None, - lat_lim: list = None, - lon_lim: list = None, - fmt: str = "%Y-%m-%d", + self, + data_source: str = "chirps", + temporal_resolution: str = "daily", + start: str = None, + end: str = None, + path: str = "", + variables: list = None, + lat_lim: list = None, + lon_lim: list = None, + fmt: str = "%Y-%m-%d", ): if data_source not in self.DataSources: - raise ValueError(f'{data_source} not supported') + raise ValueError(f"{data_source} not supported") self.datasource = self.DataSources[data_source]( start=start, @@ -39,10 +35,4 @@ def __init__( ) def download(self, progress_bar: bool = True, *args, **kwargs): - self.datasource.download( - progress_bar=progress_bar, - *args, - **kwargs - ) - - + self.datasource.download(progress_bar=progress_bar, *args, **kwargs) diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 4f1170e..8c2dce4 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -5,27 +5,31 @@ import calendar import datetime as dt import os +from pathlib import Path from typing import Dict -from loguru import logger -import yaml + import numpy as np import pandas as pd -from pathlib import Path +import yaml from ecmwfapi import ECMWFDataServer +from loguru import logger from netCDF4 import Dataset - from pyramids.raster import Raster + from earth2observe import __path__ +from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource from earth2observe.utils import print_progress_bar -from earth2observe.abstractdatasource import AbstractDataSource, AbstractCatalog + class ECMWF(AbstractDataSource): """RemoteSensing. RemoteSensing class contains methods to download ECMWF data """ + temporal_resolution = ["daily", "monthly"] spatial_resolution = 0.125 + def __init__( self, temporal_resolution: str = "daily", @@ -69,9 +73,10 @@ def __init__( ) self.path = Path(path).absolute() - - def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: str): - """check validity of input dates + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): + """check validity of input dates. Parameters ---------- @@ -98,22 +103,22 @@ def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: self.date_str = f"{self.start}/to/{self.end}" - def initialize(self): - """Initialize connection with ECMWF server""" + """Initialize connection with ECMWF server.""" try: - url = os.environ['ECMWF_API_URL'] - key = os.environ['ECMWF_API_KEY'] - email = os.environ['ECMWF_API_EMAIL'] + url = os.environ["ECMWF_API_URL"] + key = os.environ["ECMWF_API_KEY"] + email = os.environ["ECMWF_API_EMAIL"] except KeyError: - raise AuthenticationError(f"Please define the following environment variables to successfully establish a " - f"connection with ecmwf server ECMWF_API_URL, ECMWF_API_KEY, ECMWF_API_EMAIL") + raise AuthenticationError( + "Please define the following environment variables to successfully establish a " + "connection with ecmwf server ECMWF_API_URL, ECMWF_API_KEY, ECMWF_API_EMAIL" + ) self.server = ECMWFDataServer(url=url, key=key, email=email) - def create_grid(self, lat_lim: list, lon_lim: list): - """Create_grid + """Create_grid. create grid from the lat/lon boundaries @@ -135,8 +140,9 @@ def create_grid(self, lat_lim: list, lon_lim: list): lonlim_corr_two = np.ceil(lon_lim[1] / cell_size) * cell_size self.lonlim_corr = [lonlim_corr_one, lonlim_corr_two] - - def download(self, dataset: str = "interim", progress_bar: bool = True, *args, **kwargs): + def download( + self, dataset: str = "interim", progress_bar: bool = True, *args, **kwargs + ): """Download wrapper over all given variables. ECMWF method downloads ECMWF daily data for a given variable, temporal_resolution @@ -171,7 +177,10 @@ def download(self, dataset: str = "interim", progress_bar: bool = True, *args, * os.remove(del_ecmwf_dataset) def downloadDataset( - self, var_info: Dict[str, str], dataset: str = "interim", progress_bar: bool = True + self, + var_info: Dict[str, str], + dataset: str = "interim", + progress_bar: bool = True, ): """Download a climate variable. @@ -209,10 +218,8 @@ def downloadDataset( # process the downloaded data self.post_download(var_info, out_dir, dataset, progress_bar) - - def API(self, var_info, dataset): - """form the request url abd trigger the request + """form the request url abd trigger the request. Parameters ---------- @@ -288,23 +295,22 @@ def API(self, var_info, dataset): dataset=dataset, ) - @staticmethod def callAPI( - server, - output_folder: str, - download_type: str, - stream: str, - levtype: str, - param: str, - step: str, - grid: str, - time_str: str, - date_str: str, - type_str: str, - class_str: str, - area_str: str, - dataset: str = "interim", + server, + output_folder: str, + download_type: str, + stream: str, + levtype: str, + param: str, + step: str, + grid: str, + time_str: str, + date_str: str, + type_str: str, + class_str: str, + area_str: str, + dataset: str = "interim", ): """send the request to the server. @@ -367,8 +373,10 @@ def callAPI( } ) - def post_download(self, var_info: Dict[str, str], out_dir, dataset: str, progress_bar: bool=True): - """ clip the downloaded data to the extent we want + def post_download( + self, var_info: Dict[str, str], out_dir, dataset: str, progress_bar: bool = True + ): + """clip the downloaded data to the extent we want. Parameters ---------- @@ -412,7 +420,16 @@ def post_download(self, var_info: Dict[str, str], out_dir, dataset: str, progres # Define the georeference information geo_four = np.nanmax(lats) geo_one = np.nanmin(lons) - geo = tuple([geo_one, self.spatial_resolution, 0.0, geo_four, 0.0, -1 * self.spatial_resolution]) + geo = tuple( + [ + geo_one, + self.spatial_resolution, + 0.0, + geo_four, + 0.0, + -1 * self.spatial_resolution, + ] + ) # Create Waitbar if progress_bar: @@ -484,27 +501,27 @@ def post_download(self, var_info: Dict[str, str], out_dir, dataset: str, progres fh.close() + class Catalog(AbstractCatalog): - """ECMWF data catalog - This class contains the information about the ECMWF variables - http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc. - """ + """ECMWF data catalog This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc.""" + def __init__(self, version: int = 1): # get the catalog self.catalog = self.get_catalog() self.version = version - def get_catalog(self): - """readthe data catalog from disk""" + """readthe data catalog from disk.""" with open(f"{__path__[0]}/ecmwf_data_catalog.yaml", "r") as stream: catalog = yaml.safe_load(stream) return catalog def get_variable(self, var_name): - """retrieve a variable form the datasource catalog""" + """retrieve a variable form the datasource catalog.""" return self.catalog.get(var_name) + class AuthenticationError(Exception): - """Failed to establish connection with ECMWF server""" - pass \ No newline at end of file + """Failed to establish connection with ECMWF server.""" + + pass