From c85a74e15a1c5925292997c385ebf9d813dba720 Mon Sep 17 00:00:00 2001 From: Mostafa Farrag Date: Wed, 7 Dec 2022 01:58:19 +0100 Subject: [PATCH] add ecmwf data catalog --- HISTORY.rst | 2 +- README.md | 2 +- earth2observe/__init__.py | 5 - earth2observe/ecmwf.py | 516 ++++++---------------- earth2observe/ecmwf_data_catalog.yaml | 228 ++++++++++ examples/Download Satellite data.ipynb | 128 ++---- examples/{ecmwf data.py => ecmwf_data.py} | 21 +- pyproject.toml | 10 +- 8 files changed, 410 insertions(+), 502 deletions(-) create mode 100644 earth2observe/ecmwf_data_catalog.yaml rename examples/{ecmwf data.py => ecmwf_data.py} (70%) diff --git a/HISTORY.rst b/HISTORY.rst index 6d1179f..a111302 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,7 +2,7 @@ History ======= -0.1.0 (2022-05-24) +0.1.5 (2022-12-07) : add ECMWF data catalog ------------------ * First release on PyPI. diff --git a/README.md b/README.md index e7fd543..731e51b 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ pip install git+https://github.com/MAfarrag/earthobserve ## pip to install the last release you can easly use pip ``` -pip install earthobserve==0.1.3 +pip install earthobserve==0.1.5 ``` Quick start diff --git a/earth2observe/__init__.py b/earth2observe/__init__.py index a47cd93..5faea04 100644 --- a/earth2observe/__init__.py +++ b/earth2observe/__init__.py @@ -47,11 +47,6 @@ def configuration(parent_package="", top_path=None): return config -# import earth2observe.chirps as chirps -# import earth2observe.ecmwf as ecmwf -# import earth2observe.gee as gee -# import earth2observe.utils as utils - __doc__ = """ earth2observe - remote sensing package """ diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 5f4cbe3..3c9823a 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -8,10 +8,13 @@ import numpy as np import pandas as pd +import yaml from ecmwfapi import ECMWFDataServer +from loguru import logger from netCDF4 import Dataset from pyramids.raster import Raster +from earth2observe import __path__ from earth2observe.utils import print_progress_bar @@ -86,9 +89,9 @@ def __init__( self.lonlim_corr = [lonlim_corr_one, lonlim_corr_two] # TODO move it to the ECMWF method later # for ECMWF only - self.string7 = "%s/to/%s" % (self.start, self.end) + self.date_str = f"{self.start}/to/{self.end}" - def download(self, progress_bar: bool = True): + def download(self, dataset: str = "interim", progress_bar: bool = True): """ECMWF. ECMWF method downloads ECMWF daily data for a given variable, time @@ -99,6 +102,8 @@ def download(self, progress_bar: bool = True): ---------- progress_bar : TYPE, optional 0 or 1. to display the progress bar + dataset:[str] + Default is "interim" Returns ------- @@ -106,96 +111,102 @@ def download(self, progress_bar: bool = True): """ for var in self.vars: # Download data - print( - f"\nDownload ECMWF {var} data for period {self.start} till {self.end}" + logger.info( + f"Download ECMWF {var} data for period {self.start} till {self.end}" ) - - self.downloadData(var, progress_bar) # CaseParameters=[SumMean, Min, Max] + self.downloadData( + var, dataset=dataset, progress_bar=progress_bar + ) # CaseParameters=[SumMean, Min, Max] # delete the downloaded netcdf del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc") os.remove(del_ecmwf_dataset) - def downloadData(self, var: str, progress_bar: bool): + def downloadData( + self, var: str, dataset: str = "interim", progress_bar: bool = True + ): """This function downloads ECMWF six-hourly, daily or monthly data. Parameters ---------- var: [str] variable name + dataset: [str] + Default is "interm" progress_bar: [bool] True if you want to display a progress bar. """ # Load factors / unit / type of variables / accounts VarInfo = Variables(self.time) - Varname_dir = VarInfo.file_name[var] - + var_info = VarInfo.catalog.get(var) # Create Out directory - out_dir = os.path.join(self.path, self.time, Varname_dir) + out_dir = f"{self.path}, {self.time}, {VarInfo.catalog.get('file name')}" if not os.path.exists(out_dir): os.makedirs(out_dir) - DownloadType = VarInfo.DownloadType[var] - - if DownloadType == 1: - string1 = "oper" - string4 = "0" - string6 = "00:00:00/06:00:00/12:00:00/18:00:00" - string2 = "sfc" - string8 = "an" - - if DownloadType == 2: - string1 = "oper" - string4 = "12" - string6 = "00:00:00/12:00:00" - string2 = "sfc" - string8 = "fc" - - if DownloadType == 3: - string1 = "oper" - string4 = "0" - string6 = "00:00:00/06:00:00/12:00:00/18:00:00" - string2 = "pl" - string8 = "an" - - parameter_number = VarInfo.number_para[var] - - string3 = "%03d.128" % (parameter_number) - string5 = "0.125/0.125" - string9 = "ei" - string10 = "%s/%s/%s/%s" % ( - self.latlim_corr[1], - self.lonlim_corr[0], - self.latlim_corr[0], - self.lonlim_corr[1], - ) # N, W, S, E + download_type = var_info.get("download type") + # https://www.ecmwf.int/en/computing/software/ecmwf-web-api + stream = "oper" # https://apps.ecmwf.int/codes/grib/format/mars/stream/ + # step is 0, 3, 6, 9, 12 + # time_str is 0, 6, 12, 18 + # levtype: sfc for surface, pl for pressure levels (DEFAULT), pv for potential vorticity level + # type : 'fc' for forecast field, an for analysis field, ob for Observations, ai for Analysis input, + # im for Satellite images, ofb for ODB observation feedback, mfb for Monitoring feedback, + # oai for ODB analysis input, wp for Weather parameters (in BUFR format) + # key words for the API https://confluence.ecmwf.int/display/UDOC/Identification+keywords + if download_type == 1: + step = "0" + time_str = "00:00:00/06:00:00/12:00:00/18:00:00" + levtype = "sfc" # surface + type_str = "an" # analysis field + + if download_type == 2: + step = "12" + time_str = "00:00:00/12:00:00" + levtype = "sfc" # surface + type_str = "fc" # forecast field + + if download_type == 3: + step = "0" + time_str = "00:00:00/06:00:00/12:00:00/18:00:00" + levtype = "pl" # pressure levels (DEFAULT) + type_str = "an" # analysis field + + parameter_number = var_info.get("number_para") + + param = f"{parameter_number}.128" + grid = "0.125/0.125" + class_str = "ei" + # N, W, S, E + area_str = f"{self.latlim_corr[1]}/{self.lonlim_corr[0]}/{self.latlim_corr[0]}/{self.lonlim_corr[1]}" # Download data by using the ECMWF API - print("Use API ECMWF to collect the data, please wait") + logger.info("Use API ECMWF to collect the data, please wait") ECMWF.API( self.path, - DownloadType, - string1, - string2, - string3, - string4, - string5, - string6, - self.string7, - string8, - string9, - string10, + download_type, + stream, + levtype, + param, + step, + grid, + time_str, + self.date_str, + type_str, + class_str, + area_str, + dataset=dataset, ) # Open the downloaded data - NC_filename = os.path.join(self.path, "data_interim.nc") + NC_filename = os.path.join(self.path, f"data_{dataset}.nc") fh = Dataset(NC_filename, mode="r") # Get the NC variable parameter - parameter_var = VarInfo.var_name[var] - Var_unit = VarInfo.units[var] - factors_add = VarInfo.factors_add[var] - factors_mul = VarInfo.factors_mul[var] + parameter_var = var_info.get("var_name") + Var_unit = var_info.get("units") + factors_add = var_info.get("factors_add") + factors_mul = var_info.get("factors_mul") # Open the NC data Data = fh.variables[parameter_var][:] @@ -204,9 +215,9 @@ def downloadData(self, var: str, progress_bar: bool): lats = fh.variables["latitude"][:] # Define the georeference information - Geo_four = np.nanmax(lats) - Geo_one = np.nanmin(lons) - Geo_out = tuple([Geo_one, 0.125, 0.0, Geo_four, 0.0, -0.125]) + geo_four = np.nanmax(lats) + geo_one = np.nanmin(lons) + geo = tuple([geo_one, 0.125, 0.0, geo_four, 0.0, -0.125]) # Create Waitbar if progress_bar: @@ -248,24 +259,23 @@ def downloadData(self, var: str, progress_bar: bool): ) Data_one = Data[np.int_(Date_good) == 1, :, :] - # Calculate the average temperature in celcius degrees + # convert the values to the units we want Data_end = factors_mul * np.nanmean(Data_one, 0) + factors_add - if VarInfo.types[var] == "flux": + if var_info.get("types") == "flux": Data_end = Data_end * days_later - VarOutputname = VarInfo.file_name[var] + var_output_name = var_info.get("file name") # Define the out name name_out = os.path.join( out_dir, - "%s_ECMWF_ERA-Interim_%s_%s_%d.%02d.%02d.tif" - % (VarOutputname, Var_unit, self.time, year, month, day), + f"%{var_output_name}_ECMWF_ERA-Interim_{Var_unit}_{self.time}_{year}.{month}.{day}.tif", ) # Create Tiff files - # Raster.Save_as_tiff(name_out, Data_end, Geo_out, "WGS84") - Raster.createRaster(path=name_out, arr=Data_end, geo=Geo_out, epsg="WGS84") + # Raster.Save_as_tiff(name_out, Data_end, geo, "WGS84") + Raster.createRaster(path=name_out, arr=Data_end, geo=geo, epsg="WGS84") if progress_bar: amount = amount + 1 @@ -279,22 +289,21 @@ def downloadData(self, var: str, progress_bar: bool): fh.close() - return () - @staticmethod def API( - output_folder, - DownloadType, - string1, - string2, - string3, - string4, - string5, - string6, - string7, - string8, - string9, - string10, + output_folder: str, + download_type: str, + stream: str, + levtype: str, + param: str, + step: str, + grid: str, + time_str: str, + date_str: str, + type_str: str, + class_str: str, + area_str: str, + dataset: str = "interim", ): os.chdir(output_folder) @@ -302,324 +311,59 @@ def API( # url = os.environ['ECMWF_API_URL'], # key = os.environ['ECMWF_API_KEY'], # email = os.environ['ECMWF_API_EMAIL'], - if DownloadType == 1 or DownloadType == 2: + if download_type == 1 or download_type == 2: server.retrieve( { - "stream": "%s" % string1, - "levtype": "%s" % string2, - "param": "%s" % string3, - "dataset": "interim", - "step": "%s" % string4, - "grid": "%s" % string5, - "time": "%s" % string6, - "date": "%s" % string7, - "type": "%s" - % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ - "class": "%s" - % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ - "area": "%s" % string10, + "stream": stream, + "levtype": levtype, + "param": param, + "dataset": dataset, + "step": step, + "grid": grid, + "time": time_str, + "date": date_str, + "type": type_str, # http://apps.ecmwf.int/codes/grib/format/mars/type/ + "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ + "area": area_str, "format": "netcdf", - "target": "data_interim.nc", + "target": f"data_{dataset}.nc", } ) - if DownloadType == 3: + if download_type == 3: server.retrieve( { "levelist": "1000", - "stream": "%s" % string1, - "levtype": "%s" % string2, - "param": "%s" % string3, - "dataset": "interim", - "step": "%s" % string4, - "grid": "%s" % string5, - "time": "%s" % string6, - "date": "%s" % string7, - "type": "%s" - % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ - "class": "%s" - % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ - "area": "%s" % string10, + "stream": stream, + "levtype": levtype, + "param": param, + "dataset": dataset, + "step": step, + "grid": grid, + "time": time_str, + "date": date_str, + "type": type_str, # http://apps.ecmwf.int/codes/grib/format/mars/type/ + "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ + "area": area_str, "format": "netcdf", - "target": "data_interim.nc", + "target": f"data_{dataset}.nc", } ) - return () - class Variables: """This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc.""" - number_para = { - "T": 130, - "2T": 167, - "SRO": 8, - "SSRO": 9, - "WIND": 10, - "10SI": 207, - "SP": 134, - "Q": 133, - "SSR": 176, - "R": 157, - "E": 182, - "SUND": 189, - "RO": 205, - "TP": 228, - "10U": 165, - "10V": 166, - "2D": 168, - "SR": 173, - "AL": 174, - "HCC": 188, - } - - var_name = { - "T": "t", - "2T": "t2m", - "SRO": "sro", - "SSRO": "ssro", - "WIND": "wind", - "10SI": "10si", - "SP": "sp", - "Q": "q", - "SSR": "ssr", - "R": "r", - "E": "e", - "SUND": "sund", - "RO": "ro", - "TP": "tp", - "10U": "u10", - "10V": "v10", - "2D": "d2m", - "SR": "sr", - "AL": "al", - "HCC": "hcc", - } - - # ECMWF data - descriptions = { - "T": "Temperature [K]", - "2T": "2 meter Temperature [K]", - "SRO": "Surface Runoff [m]", - "SSRO": "Sub-surface Runoff [m]", - "WIND": "Wind speed [m s-1]", - "10SI": "10 metre windspeed [m s-1]", - "SP": "Surface Pressure [pa]", - "Q": "Specific humidity [kg kg-1]", - "SSR": "Surface solar radiation [W m-2 s]", - "R": "Relative humidity [%]", - "E": "Evaporation [m of water]", - "SUND": "Sunshine duration [s]", - "RO": "Runoff [m]", - "TP": "Total Precipitation [m]", - "10U": "10 metre U wind component [m s-1]", - "10V": "10 metre V wind component [m s-1]", - "2D": "2 metre dewpoint temperature [K]", - "SR": "Surface roughness [m]", - "AL": "Albedo []", - "HCC": "High cloud cover []", - } - - # Factor add to get output - factors_add = { - "T": -273.15, - "2T": -273.15, - "SRO": 0, - "SSRO": 0, - "WIND": 0, - "10SI": 0, - "SP": 0, - "Q": 0, - "SSR": 0, - "R": 0, - "E": 0, - "SUND": 0, - "RO": 0, - "TP": 0, - "10U": 0, - "10V": 0, - "2D": -273.15, - "SR": 0, - "AL": 0, - "HCC": 0, - } - - # Factor multiply to get output - factors_mul = { - "T": 1, - "2T": 1, - "SRO": 1000, - "SSRO": 1000, - "WIND": 1, - "10SI": 1, - "SP": 0.001, - "Q": 1, - "SSR": 1, - "R": 1, - "E": 1000, - "SUND": 1, - "RO": 1000, - "TP": 1000, - "10U": 1, - "10V": 1, - "2D": 1, - "SR": 1, - "AL": 1, - "HCC": 1, - } - - types = { - "T": "state", - "2T": "state", - "SRO": "flux", - "SSRO": "flux", - "WIND": "state", - "10SI": "state", - "SP": "state", - "Q": "state", - "SSR": "state", - "R": "state", - "E": "flux", - "SUND": "flux", - "RO": "flux", - "TP": "flux", - "10U": "state", - "10V": "state", - "2D": "state", - "SR": "state", - "AL": "state", - "HCC": "state", - } - - file_name = { - "T": "Tair2m", - "2T": "Tair", - "SRO": "Surf_Runoff", - "SSRO": "Subsurf_Runoff", - "WIND": "Wind", - "10SI": "Wind10m", - "SP": "Psurf", - "Q": "Qair", - "SSR": "SWnet", - "R": "RelQair", - "E": "Evaporation", - "SUND": "SunDur", - "RO": "Runoff", - "TP": "P", - "10U": "Wind_U", - "10V": "Wind_V", - "2D": "Dewpoint2m", - "SR": "SurfRoughness", - "AL": "Albedo", - "HCC": "HighCloudCover", - } - - DownloadType = { - "T": 3, - "2T": 1, - "SRO": 0, - "SSRO": 0, - "WIND": 0, - "10SI": 0, - "SP": 1, - "Q": 3, - "SSR": 2, - "R": 3, - "E": 2, - "SUND": 2, - "RO": 2, - "TP": 2, - "10U": 1, - "10V": 1, - "2D": 1, - "SR": 1, - "AL": 1, - "HCC": 1, - } - - def __init__(self, step): - - # output units after applying factor - if step == "six_hourly": - self.units = { - "T": "C", - "2T": "C", - "SRO": "mm", - "SSRO": "mm", - "WIND": "m_s-1", - "10SI": "m_s-1", - "SP": "kpa", - "Q": "kg_kg-1", - "SSR": "W_m-2_s", - "R": "percentage", - "E": "mm", - "SUND": "s", - "RO": "mm", - "TP": "mm", - "10U": "m_s-1", - "10V": "m_s-1", - "2D": "C", - "SR": "m", - "AL": "-", - "HCC": "-", - } - - elif step == "daily": - self.units = { - "T": "C", - "2T": "C", - "SRO": "mm", - "SSRO": "mm", - "WIND": "m_s-1", - "10SI": "m_s-1", - "SP": "kpa", - "Q": "kg_kg-1", - "SSR": "W_m-2_s", - "R": "percentage", - "E": "mm", - "SUND": "s", - "RO": "mm", - "TP": "mm", - "10U": "m_s-1", - "10V": "m_s-1", - "2D": "C", - "SR": "m", - "AL": "-", - "HCC": "-", - } - - elif step == "monthly": - self.units = { - "T": "C", - "2T": "C", - "SRO": "mm", - "SSRO": "mm", - "WIND": "m_s-1", - "10SI": "m_s-1", - "SP": "kpa", - "Q": "kg_kg-1", - "SSR": "W_m-2_s", - "R": "percentage", - "E": "mm", - "SUND": "s", - "RO": "mm", - "TP": "mm", - "10U": "m_s-1", - "10V": "m_s-1", - "2D": "C", - "SR": "m", - "AL": "-", - "HCC": "-", - } - - else: - raise KeyError("The input time step is not supported") - - def __str__(self): - print( - f"Variable name:\n {self.var_name}\nDescriptions\n{self.descriptions}\nUnits : \n{self.units}" - ) + with open(f"{__path__[0]}/ecmwf_data_catalog.yaml", "r") as stream: + catalog = yaml.safe_load(stream) + + def __init__(self, step, version: int = 1): + self.version = version + + # def __str__(self): + # print( + # f"Variable name:\n {self.var_name}\nDescriptions\n{self.descriptions}\nUnits : \n{self.units}" + # ) def ListAttributes(self): """Print Attributes List.""" diff --git a/earth2observe/ecmwf_data_catalog.yaml b/earth2observe/ecmwf_data_catalog.yaml new file mode 100644 index 0000000..63d2699 --- /dev/null +++ b/earth2observe/ecmwf_data_catalog.yaml @@ -0,0 +1,228 @@ +version: 1 +# parameter database +#https://apps.ecmwf.int/codes/grib/param-db/?filter=netcdf +# units is the units after applying factor +# available datasets +#https://confluence.ecmwf.int/display/WEBAPI/Available+ECMWF+Public+Datasets +datasets: [cams_gfas, cams_nrealtime, cera20c, cera_sat, era15, era20c, era20cm, era20cmv0, era40, geff_reanalysis, icoads, interim, interim_land, ispd, macc, macc_nrealtime, s2s, tigge, uerra, yopp, yotc] + +T: + descriptions: Temperature [K] + units: C + types: state + temporal resolution: [six hours, daily, monthly] + file name: Tair2m + download type: 3 + number_para: 130 + var_name: t + factors_add: -273.15 + factors_mul: 1 +2T: + descriptions: 2 meter Temperature [K] + units: C + types: state + temporal resolution: [six hours, daily, monthly] + file name: Tair + download type: 1 + number_para: 167 + var_name: t2m + factors_add: -273.15 + factors_mul: 1 +SRO: + descriptions: Surface Runoff [m] + units: mm + types: flux + temporal resolution: [six hours, daily, monthly] + file name: Surf_Runoff + download type: 0 + number_para: 8 + var_name: sro + factors_add: 0 + factors_mul: 1000 +SSRO: + descriptions: Sub-surface Runoff [m] + units: mm + types: flux + temporal resolution: [six hours, daily, monthly] + file name: Subsurf_Runoff + download type: 0 + number_para: 9 + var_name: ssro + factors_add: 0 + factors_mul: 1000 +WIND: + descriptions: Wind speed [m s-1] + units: m_s-1 + types: state + temporal resolution: [six hours, daily, monthly] + file name: Wind + download type: 0 + number_para: 10 + var_name: wind + factors_add: 0 + factors_mul: 1 +10SI: + descriptions: 10 metre windspeed [m s-1] + units: m_s-1 + types: state + temporal resolution: [six hours, daily, monthly] + file name: Wind10m + download type: 0 + number_para: 207 + var_name: 10si + factors_add: 0 + factors_mul: 1 +SP: + descriptions: Surface Pressure [pa] + units: kpa + types: state + temporal resolution: [six hours, daily, monthly] + file name: Psurf + download type: 1 + number_para: 134 + var_name: sp + factors_add: 0 + factors_mul: 0.001 +Q: + descriptions: Specific humidity [kg kg-1] + units: kg_kg-1 + types: state + temporal resolution: [six hours, daily, monthly] + file name: Qair + download type: 3 + number_para: 133 + var_name: q + factors_add: 0 + factors_mul: 1 +SSR: + descriptions: Surface solar radiation [W m-2 s] + units: W_m-2_s + types: state + temporal resolution: [six hours, daily, monthly] + file name: SWnet + download type: 2 + number_para: 176 + var_name: ssr + factors_add: 0 + factors_mul: 1 +R: + descriptions: Relative humidity [%] + units: percentage + types: state + temporal resolution: [six hours, daily, monthly] + file name: RelQair + download type: 3 + number_para: 157 + var_name: r + factors_add: 0 + factors_mul: 1 +E: + descriptions: Evaporation [m of water] + units: mm + types: flux + temporal resolution: [six hours, daily, monthly] + file name: Evaporation + download type: 2 + number_para: 182 + var_name: e + factors_add: 0 + factors_mul: 1000 +SUND: + descriptions: Sunshine duration [s] + units: s + types: flux + temporal resolution: [six hours, daily, monthly] + file name: SunDur + download type: 2 + number_para: 189 + var_name: sund + factors_add: 0 + factors_mul: 1 +RO: + descriptions: Runoff [m] + units: mm + types: flux + temporal resolution: [six hours, daily, monthly] + file name: Runoff + download type: 2 + number_para: 205 + var_name: ro + factors_add: 0 + factors_mul: 1000 +TP: + descriptions: Total Precipitation [m] + units: mm + types: flux + temporal resolution: [six hours, daily, monthly] + file name: P + download type: 2 + number_para: 228 + var_name: tp + factors_add: 0 + factors_mul: 1000 +10U: + descriptions: 10 metre U wind component [m s-1] + units: m_s-1 + types: state + temporal resolution: [six hours, daily, monthly] + file name: Wind_U + download type: 1 + number_para: 165 + var_name: u10 + factors_add: 0 + factors_mul: 1 +10V: + descriptions: 10 metre V wind component [m s-1] + units: m_s-1 + types: state + temporal resolution: [six hours, daily, monthly] + file name: Wind_V + download type: 1 + number_para: 166 + var_name: v10 + factors_add: 0 + factors_mul: 1 +2D: + descriptions: 2 metre dewpoint temperature [K] + units: C + types: state + temporal resolution: [six hours, daily, monthly] + file name: Dewpoint2m + download type: 1 + number_para: 168 + var_name: d2m + factors_add: -273.15 + factors_mul: 1 +SR: + descriptions: Surface roughness [m] + units: m + types: state + temporal resolution: [six hours, daily, monthly] + file name: SurfRoughness + download type: 1 + number_para: 173 + var_name: sr + factors_add: 0 + factors_mul: 1 +AL: + descriptions: Albedo [] + units: unitless + types: state + temporal resolution: [six hours, daily, monthly] + file name: Albedo + download type: 1 + number_para: 174 + var_name: al + factors_add: 0 + factors_mul: 1 +HCC: + descriptions: High cloud cover [] + units: unitless + types: state + temporal resolution: [six hours, daily, monthly] + file name: HighCloudCover + download type: 1 + number_para: 188 + var_name: hcc + factors_add: 0 + factors_mul: 1 diff --git a/examples/Download Satellite data.ipynb b/examples/Download Satellite data.ipynb index 2448ba0..b3f6128 100644 --- a/examples/Download Satellite data.ipynb +++ b/examples/Download Satellite data.ipynb @@ -3,11 +3,7 @@ { "cell_type": "markdown", "id": "48fff0da-2d7a-49ea-ba13-d4768a89bae0", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "# Download Satellite data" ] @@ -15,11 +11,7 @@ { "cell_type": "markdown", "id": "2c773dd4-6dc6-4c86-aeed-ad82b6118ccb", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "## ECMWF" ] @@ -27,11 +19,7 @@ { "cell_type": "markdown", "id": "7ac42c26-f454-40e5-8dc2-668efbc2d02b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "### Installation of ECMWF API key" ] @@ -39,11 +27,7 @@ { "cell_type": "markdown", "id": "ad7c9c5c-ed4c-41b7-afdb-40fb471e015b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "1 - to be able to use Hapi to download ECMWF data you need to register and setup your account in the ECMWF website (https://apps.ecmwf.int/registration/)\n", "\n", @@ -54,11 +38,7 @@ { "cell_type": "markdown", "id": "f3d80671-55eb-4915-990b-78f1dcc4e0d1", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "### Using ResmoteSensing module from Hapi " ] @@ -67,27 +47,17 @@ "cell_type": "code", "execution_count": 1, "id": "9d733330-95bb-42b4-b6f5-895119345222", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ "from earth2observe.ecmwf import ECMWF\n", - "from earth2observe.ecmwf import Variables\n", - "import os\n", - "#os.chdir(\"F:/01Algorithms/Hydrology/HAPI/Examples\")" + "from earth2observe.ecmwf import Variables" ] }, { "cell_type": "markdown", "id": "bc750181-4a25-48f1-b510-ee834484b7c9", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "For the information about the ECMWF data https://apps.ecmwf.int/codes/grib/param-db/\n", "ECMWP data are represented as variables to know the name of the variable you want to download you can check the object `Variables`\n", @@ -99,11 +69,7 @@ "cell_type": "code", "execution_count": 3, "id": "35bb343d-4354-43e2-82ea-6ebf14aa3615", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -126,11 +92,7 @@ { "cell_type": "markdown", "id": "0a2a6ca8-a09b-47b8-b97e-8d1c1e11c561", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "### Inputs" ] @@ -138,11 +100,7 @@ { "cell_type": "markdown", "id": "be829713-e3bb-442e-a2b1-5dfffb46b028", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "- After selecting the variable, temperature and evapotranspiration ['E','T']\n", "\n", @@ -156,19 +114,15 @@ "cell_type": "code", "execution_count": 4, "id": "c73ad20b-e8cd-4a0e-9107-12a2cfbac207", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ - "StartDate = '2009-01-01'\n", - "EndDate = '2009-01-10'\n", - "Time = 'daily'\n", + "start_date = '2009-01-01'\n", + "end_date = '2009-01-10'\n", + "temporal_resolution = 'daily'\n", "lat = [4.190755, 4.643963]\n", "lon = [-75.649243, -74.727286]\n", - "Path = \"/data/satellite_data/\"\n", + "path = \"/data/satellite_data/\"\n", "# Temperature, Evapotranspiration\n", "variables = ['T', 'E']" ] @@ -177,11 +131,7 @@ "cell_type": "code", "execution_count": 6, "id": "405220ce-2d9b-4cee-9137-ee639153ef64", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -359,8 +309,8 @@ } ], "source": [ - "Coello = ECMWF(start=StartDate, end=EndDate, time=Time,\n", - " lat_lim=lat, lon_lim=lon, path=Path, variables=variables)\n", + "Coello = ECMWF(start=start_date, end=end_date, time=temporal_resolution,\n", + " lat_lim=lat, lon_lim=lon, path=path, variables=variables)\n", "\n", "Coello.download(Waitbar=1)" ] @@ -368,11 +318,7 @@ { "cell_type": "markdown", "id": "195e57ae-6345-45a4-a461-865734bafd73", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "## CHIRPS" ] @@ -380,11 +326,7 @@ { "cell_type": "markdown", "id": "358e0726-28fb-4200-9674-9d0754b308cd", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "Using the same inputs (period and extent) CHRIPS data does not deen any registration" ] @@ -393,11 +335,7 @@ "cell_type": "code", "execution_count": 7, "id": "bf101450-3e00-4572-a50c-673a267996c7", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ "from earth2observe.chirps import CHIRPS" @@ -407,11 +345,7 @@ "cell_type": "code", "execution_count": 8, "id": "e9ab20a1-786f-4692-913e-94f2f85b0281", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -432,19 +366,15 @@ } ], "source": [ - "Coello = CHIRPS(start=StartDate, end=EndDate, time=Time,\n", - " lat_lim=lat, lon_lim=lon, path=Path)\n", + "Coello = CHIRPS(start=start_date, end=end_date, time=temporal_resolution,\n", + " lat_lim=lat, lon_lim=lon, path=path)\n", "Coello.Download()" ] }, { "cell_type": "markdown", "id": "2a2dbda9-82f9-44c3-9d0f-c9b330ae880a", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "### Parallel download\n", "- As the CHRIPS data are downloaded directly from ftp server, so several downloads can be done at the same time\n", @@ -456,11 +386,7 @@ "cell_type": "code", "execution_count": null, "id": "9d0cfa73-938e-4334-b8cf-9ebc50ddaa81", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ "Coello.Download(cores=4)" diff --git a/examples/ecmwf data.py b/examples/ecmwf_data.py similarity index 70% rename from examples/ecmwf data.py rename to examples/ecmwf_data.py index cb7cfed..07078c9 100644 --- a/examples/ecmwf data.py +++ b/examples/ecmwf_data.py @@ -4,10 +4,12 @@ 2 - Install ECMWF key (instruction are here https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key) """ +import os from earth2observe.ecmwf import ECMWF, Variables -path = r"examples\data\ecmwf" +rpath = os.getcwd() +path = rf"{rpath}\examples\data\ecmwf" start = "2009-01-01" end = "2009-02-01" @@ -16,10 +18,10 @@ lonlim = [-75.65, -74.73] # %% Vars = Variables("daily") -Vars.__str__() +print(Vars.catalog) # %% # Temperature, Evapotranspiration -variables = ["T", "E"] +variables = ["E"] # "T", Coello = ECMWF( time=time, @@ -31,4 +33,17 @@ variables=variables, ) +Coello.download(dataset="interim") +#%% +variables = ["SRO"] +Coello = ECMWF( + time=time, + start=start, + end=end, + lat_lim=latlim, + lon_lim=lonlim, + path=path, + variables=variables, +) + Coello.download() diff --git a/pyproject.toml b/pyproject.toml index 0f4e11b..1b5a274 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "earth2observe" -version = "0.1.4" +version = "0.1.5" description = "remote sensing package" authors = ["Mostafa Farrag "] license = "GNU General Public License v3" @@ -32,11 +32,11 @@ ecmwf-api-client = "^1.6.3" earthengine-api = "^0.1.324" loguru = "^0.6.0" joblib = "^1.2.0" -gdal = "3.4.3" -#gdal = {url = "https://download.lfd.uci.edu/pythonlibs/archived/GDAL-3.4.3-cp39-cp39-win_amd64.whl" } +#gdal = "3.4.3" +gdal = {url = "https://download.lfd.uci.edu/pythonlibs/archived/GDAL-3.4.3-cp39-cp39-win_amd64.whl" } pyramids-gis = "^0.2.6" -Fiona = "1.8.21" -#fiona = {url = "https://download.lfd.uci.edu/pythonlibs/archived/Fiona-1.8.21-cp39-cp39-win_amd64.whl" } +#Fiona = "1.8.21" +fiona = {url = "https://download.lfd.uci.edu/pythonlibs/archived/Fiona-1.8.21-cp39-cp39-win_amd64.whl" }