Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@MAfarrag
MAfarrag committed Jan 15, 2023
1 parent bed731f commit 778c1f7
Show file tree
Hide file tree
Showing 4 changed files with 153 additions and 135 deletions.
41 changes: 19 additions & 22 deletions earth2observe/abstractdatasource.py
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
from typing import Dict
from abc import ABC, abstractmethod
from typing import Dict


class AbstractDataSource(ABC):
"""
Bluebrint for all class for different datasources
"""
"""Bluebrint for all class for different datasources."""

def __init__(
self,
start: str = None,
end: str = None,
variables: list = None,
temporal_resolution: str = "daily",
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
# path: str = "",
self,
start: str = None,
end: str = None,
variables: list = None,
temporal_resolution: str = "daily",
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
# path: str = "",
):
"""
Expand Down Expand Up @@ -50,7 +49,7 @@ def __init__(

@abstractmethod
def check_input_dates(self):
"""Check validity of input dates"""
"""Check validity of input dates."""
pass

@abstractmethod
Expand All @@ -60,7 +59,7 @@ def initialize(self):

@abstractmethod
def create_grid(self):
"""create a grid from the lat/lon boundaries"""
"""create a grid from the lat/lon boundaries."""
pass

@abstractmethod
Expand All @@ -70,29 +69,27 @@ def download(self):
# list of dates
pass


# @abstractmethod
def downloadDataset(self):
"""Download single variable/dataset"""
"""Download single variable/dataset."""
# used for non ftp servers
pass

@abstractmethod
def API(self):
"""send/recieve request to the dataset server"""
"""send/recieve request to the dataset server."""
pass



class AbstractCatalog(ABC):
"""abstrach class for the datasource catalog"""
"""abstrach class for the datasource catalog."""

@abstractmethod
def get_catalog(self):
"""read the catalog of the datasource from disk or retrieve it from server"""
"""read the catalog of the datasource from disk or retrieve it from server."""
pass

@abstractmethod
def get_variable(self, var_name) -> Dict[str, str]:
"""get the details of a specific variable"""
"""get the details of a specific variable."""
pass
86 changes: 50 additions & 36 deletions earth2observe/chirps.py
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
from typing import List
import datetime as dt
import os
from pathlib import Path
from ftplib import FTP
from pathlib import Path

import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from osgeo import gdal
from pyramids.raster import Raster
from pyramids.utils import extractFromGZ
from earth2observe.abstractdatasource import AbstractDataSource, AbstractCatalog

from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource
from earth2observe.utils import print_progress_bar


class CHIRPS(AbstractDataSource):
"""CHIRPS."""

api_url: str = "data.chc.ucsb.edu"
start_date: str = "1981-01-01"
end_date: str = "Now"
Expand All @@ -26,15 +27,15 @@ class CHIRPS(AbstractDataSource):
clipped_fname = "P_CHIRPS.v2.0"

def __init__(
self,
temporal_resolution: str = "daily",
start: str = None,
end: str = None,
path: str = "",
variables: list = None,
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
self,
temporal_resolution: str = "daily",
start: str = None,
end: str = None,
path: str = "",
variables: list = None,
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
):
"""CHIRPS.
Expand Down Expand Up @@ -66,16 +67,18 @@ def __init__(
lon_lim=lon_lim,
fmt=fmt,
)
self.output_folder = os.path.join(Path(path).absolute(), "chirps", "precipitation")
self.output_folder = os.path.join(
Path(path).absolute(), "chirps", "precipitation"
)

# make directory if it not exists
if not os.path.exists(self.output_folder):
os.makedirs(self.output_folder)



def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt: str):
"""check validity of input dates
def check_input_dates(
self, start: str, end: str, temporal_resolution: str, fmt: str
):
"""check validity of input dates.
Parameters
----------
Expand Down Expand Up @@ -115,12 +118,12 @@ def check_input_dates(self, start: str, end: str, temporal_resolution: str, fmt:
self.dates = pd.date_range(self.start, self.end, freq=self.time_freq)

def initialize(self):
"""Initialize FTP server"""
"""Initialize FTP server."""
print("FTP server datasources does not need server initialization")
pass

def create_grid(self, lat_lim: list, lon_lim: list):
"""Create_grid
"""Create_grid.
create grid from the lat/lon boundaries
Expand Down Expand Up @@ -166,7 +169,6 @@ def create_grid(self, lat_lim: list, lon_lim: list):
)
)


def download(self, progress_bar: bool = True, cores=None, *args, **kwargs):
"""Download.
Expand Down Expand Up @@ -228,14 +230,13 @@ def download(self, progress_bar: bool = True, cores=None, *args, **kwargs):
return results

def API(self, date, args):
"""form the request url abd trigger the request
"""form the request url abd trigger the request.
Parameters
----------
date:
args: [list]
"""
[output_folder, temp_resolution, xID, yID, lon_lim, latlim] = args

Expand All @@ -252,28 +253,31 @@ def API(self, date, args):
filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif.gz"
outfilename = os.path.join(
output_folder,
f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif"
f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif",
)
DirFileEnd = os.path.join(
output_folder,
f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif"
f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif",
)
elif temp_resolution == "monthly":
filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz"
filename = (
f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz"
)
outfilename = os.path.join(
output_folder,
f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif"
f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif",
)
DirFileEnd = os.path.join(
output_folder,
f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif"
f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif",
)
else:
raise KeyError("The input temporal_resolution interval is not supported")

self.callAPI(pathFTP, output_folder, filename)
self.post_download(output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd)

self.post_download(
output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd
)

@staticmethod
def callAPI(pathFTP: str, output_folder: str, filename: str):
Expand Down Expand Up @@ -314,9 +318,18 @@ def callAPI(pathFTP: str, output_folder: str, filename: str):
ftp.retrbinary("RETR " + filename, lf.write, 8192)
lf.close()


def post_download(self, output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd):
"""clip the downloaded data to the extent we want
def post_download(
self,
output_folder,
filename,
lon_lim,
latlim,
xID,
yID,
outfilename,
DirFileEnd,
):
"""clip the downloaded data to the extent we want.
Parameters
----------
Expand Down Expand Up @@ -379,24 +392,25 @@ def listAttributes(self):

print("\n")


class Catalog(AbstractCatalog):
""" CHIRPS data catalog"""
"""CHIRPS data catalog."""

def __init__(self):
self.catalog = self.get_catalog()

def get_catalog(self):
"""return the catalog"""
"""return the catalog."""
return {
"Precipitation": {
"descriptions": "rainfall [mm/temporal_resolution step]",
"units": "mm/temporal_resolution step",
"temporal resolution": ["daily", "monthly"],
"file name": "rainfall",
"var_name": "R",
}
}
}

def get_variable(self, var_name):
"""get the details of a specific variable"""
"""get the details of a specific variable."""
return self.catalog.get(var_name)
44 changes: 17 additions & 27 deletions earth2observe/earth2observe.py
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,27 @@
from earth2observe.ecmwf import ECMWF
"""Front end module that runs each data source backend."""
from earth2observe.chirps import CHIRPS
from earth2observe.ecmwf import ECMWF

class Earth2Observe:
"""
End user class to call all the data source classes abailable in earth2observe.
"""

DataSources = {
"ecmwf": ECMWF,
"chirps": CHIRPS
}
class Earth2Observe:
"""End user class to call all the data source classes abailable in earth2observe."""

DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS}

def __init__(
self,
data_source: str = "chirps",
temporal_resolution: str = "daily",
start: str = None,
end: str = None,
path: str = "",
variables: list = None,
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
self,
data_source: str = "chirps",
temporal_resolution: str = "daily",
start: str = None,
end: str = None,
path: str = "",
variables: list = None,
lat_lim: list = None,
lon_lim: list = None,
fmt: str = "%Y-%m-%d",
):
if data_source not in self.DataSources:
raise ValueError(f'{data_source} not supported')
raise ValueError(f"{data_source} not supported")

self.datasource = self.DataSources[data_source](
start=start,
Expand All @@ -39,10 +35,4 @@ def __init__(
)

def download(self, progress_bar: bool = True, *args, **kwargs):
self.datasource.download(
progress_bar=progress_bar,
*args,
**kwargs
)


self.datasource.download(progress_bar=progress_bar, *args, **kwargs)
Loading

0 comments on commit 778c1f7

Please sign in to comment.