diff --git a/.coveralls.yml b/.coveralls.yml index 84b054c..06e982e 100644 --- a/.coveralls.yml +++ b/.coveralls.yml @@ -1,3 +1,2 @@ repo_token: hxJrvjqiH2xBI7eit7BAb7FidH0LeYpGq service_name: github-action - diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9f24714..eda6933 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -48,11 +48,11 @@ jobs: # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. - + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild @@ -61,7 +61,7 @@ jobs: # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - # If the Autobuild fails above, remove it and uncomment the following three lines. + # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d19bf87..8f19042 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -62,4 +62,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage1.xml,./coverage2.xml - directory: ./coverage/reports/ \ No newline at end of file + directory: ./coverage/reports/ diff --git a/.gitignore b/.gitignore index f65edba..dc7684d 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ build_artifacts mo_* conda/ /examples/data/ecmwf/daily/* +.idea/* +poetry.lock +pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5450999..e3bc9b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,12 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 - hooks: -# - id: check-yaml -# name: "[yaml - check] validate yaml" - - id: end-of-file-fixer - name: "[py - check] validate yaml" - - id: trailing-whitespace - name: "[file - format] trim trailing whitespace" - args: [ --markdown-linebreak-ext=md ] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + name: "[py - check] validate yaml" + - id: trailing-whitespace + name: "[file - format] trim trailing whitespace" + args: [ --markdown-linebreak-ext=md ] - id: check-added-large-files name: "[file - check] large file" args: [ --maxkb=5000 ] @@ -37,41 +35,51 @@ repos: --no-sort-keys ] - id: requirements-txt-fixer name: "[reqs - format] fix requirements.txt" + - id: check-yaml + name: "[yaml - check] validate yaml" +- repo: https://github.com/PyCQA/docformatter + rev: v1.4 + hooks: + - id: docformatter + name: "[py - format] docformatter" + args: [ -i, --wrap-summaries, "0" ] -# - repo: https://github.com/PyCQA/docformatter -# rev: v1.4 -# hooks: -# - id: docformatter -## name: "[py - format] docformatter" -## args: [ -i, --wrap-summaries, "0" ] +- repo: https://github.com/PyCQA/pydocstyle + rev: 6.1.1 + hooks: + - id: pydocstyle + name: "[py - check] pydocstyle" + files: ^Hapi/ -# - repo: https://github.com/PyCQA/pydocstyle -# rev: 6.1.1 -# hooks: -# - id: pydocstyle -# name: "[py - check] pydocstyle" -# files: ^Hapi/ - # TODO : uncheck later and fix all the problems of line too long -#- repo: https://gitlab.com/pycqa/flake8 -# rev: 3.8.4 -# hooks: -# - id: flake8 -# name: "[py - check] flake8" -# language_version: python3.8 -# TODO : this hook does not fix the files - #- repo: https://github.com/psf/black -# rev: 19.3b0 -# hooks: -# - id: black -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.7.0 - hooks: - - id: isort - name: "[py - format] isort" +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + name: "[py - check] flake8" + language_version: python3.9 + exclude: ^(examples/|tests/) -#- repo: https://github.com/ambv/black -# rev: 20.8b1 -# hooks: -# - id: black -# name: "[py - format] black" -# language_version: python3.8 +- repo: https://github.com/psf/black + rev: 22.8.0 + hooks: + - id: black +- repo: https://github.com/pre-commit/mirrors-isort + rev: v5.7.0 + hooks: + - id: isort + name: "[py - format] isort" +- repo: https://github.com/ambv/black + rev: 22.8.0 + hooks: + - id: black + name: "[py - format] black" + language_version: python3.9 + +- repo: local + hooks: + - id: pytest-check + name: pytest-check + entry: pytest + language: system + pass_filenames: false + always_run: true diff --git a/README.md b/README.md index 05262a8..0799257 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,12 @@ earthobserve Main Features ------------- - - + - Future work ------------- - - + - diff --git a/clone.json b/clone.json index ed40d55..84f855d 100644 --- a/clone.json +++ b/clone.json @@ -1,4 +1,4 @@ { - "message": "Must have push access to repository", - "documentation_url": "https://docs.github.com/rest/reference/repos#get-repository-clones" + "message": "Must have push access to repository", + "documentation_url": "https://docs.github.com/rest/reference/repos#get-repository-clones" } diff --git a/docs/conf.py b/docs/conf.py index bb14dd7..11e7682 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,18 +9,17 @@ # All configuration values have a default; values that are commented out # serve to show the default. -# General information about the project. -project = u"pyramids" -author = "Mostafa Farrag" - -# copyright = u"2013-2019, " - - import os import sys # import sphinx_rtd_theme +# General information about the project. +project = "pyramids" +author = "Mostafa Farrag" + +# copyright = u"2013-2019, " + html_theme = "sphinxdoc" # html_theme = "agogo" html_theme_path = ["."] @@ -207,7 +206,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ("index", "pyramids.tex", u"pyramids Documentation", u"Mostafa Farrag", "report") + ("index", "pyramids.tex", "pyramids Documentation", "Mostafa Farrag", "report") ] # The name of an image file (relative to this directory) to place at the top of @@ -250,8 +249,8 @@ ( "index", "pyramids", - u"pyramids Documentation", - u"Mostafa Farrag", + "pyramids Documentation", + "Mostafa Farrag", "pyramids", "One line description of project.", "Miscellaneous", diff --git a/earth2observe/__init__.py b/earth2observe/__init__.py index 6864b85..a47cd93 100644 --- a/earth2observe/__init__.py +++ b/earth2observe/__init__.py @@ -13,7 +13,7 @@ # documentation format __author__ = "Mostafa Farrag" -__email__ = 'moah.farag@gmail.com' +__email__ = "moah.farag@gmail.com" __docformat__ = "restructuredtext" # Let users know if they're missing any of our hard dependencies @@ -25,15 +25,17 @@ __import__(dependency) except ImportError as e: missing_dependencies.append(dependency) + print(e) if missing_dependencies: raise ImportError("Missing required dependencies {0}".format(missing_dependencies)) -def configuration(parent_package='',top_path=None): + +def configuration(parent_package="", top_path=None): from numpy.distutils.misc_util import Configuration - config = Configuration(None,parent_package,top_path) + config = Configuration(None, parent_package, top_path) config.set_options( ignore_setup_xxx_py=True, assume_default_configuration=True, @@ -41,14 +43,14 @@ def configuration(parent_package='',top_path=None): quiet=True, ) - config.add_subpackage('gee') + config.add_subpackage("gee") return config -import earth2observe.chirps as chirps -import earth2observe.ecmwf as ecmwf -import earth2observe.gee as gee -import earth2observe.utils as utils +# import earth2observe.chirps as chirps +# import earth2observe.ecmwf as ecmwf +# import earth2observe.gee as gee +# import earth2observe.utils as utils __doc__ = """ earth2observe - remote sensing package diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index d0eb191..56d87a7 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -1,27 +1,28 @@ -import os import datetime as dt +import os +from ftplib import FTP + import numpy as np import pandas as pd -from ftplib import FTP from joblib import Parallel, delayed from osgeo import gdal from pyramids.raster import Raster -from earth2observe.utils import print_progress_bar, extractFromGZ + +from earth2observe.utils import extractFromGZ, print_progress_bar class CHIRPS: - """ - CHIRPS - """ + """CHIRPS.""" + def __init__( - self, - start: str="", - end: str="", - lat_lim: list=[], - lon_lim: list=[], - time: str="daily", - path: str="", - fmt: str="%Y-%m-%d", + self, + start: str = "", + end: str = "", + lat_lim: list = [], + lon_lim: list = [], + time: str = "daily", + path: str = "", + fmt: str = "%Y-%m-%d", ): """CHIRPS. @@ -52,7 +53,9 @@ def __init__( self.output_folder = os.path.join(path, "Precipitation", "CHIRPS", "Daily") elif time == "monthly": self.time_freq = "MS" - self.output_folder = os.path.join(path, "Precipitation", "CHIRPS", "Monthly") + self.output_folder = os.path.join( + path, "Precipitation", "CHIRPS", "Monthly" + ) else: raise KeyError("The input time interval is not supported") self.time = time @@ -94,7 +97,9 @@ def __init__( self.lon_lim = lon_lim # Define IDs self.yID = 2000 - np.int16( - np.array([np.ceil((lat_lim[1] + 50) * 20), np.floor((lat_lim[0] + 50) * 20)]) + np.array( + [np.ceil((lat_lim[1] + 50) * 20), np.floor((lat_lim[0] + 50) * 20)] + ) ) self.xID = np.int16( np.array( @@ -102,8 +107,7 @@ def __init__( ) ) - - def Download(self, progress_bar: bool=True, cores=None): + def Download(self, progress_bar: bool = True, cores=None): """Download. Download method downloads CHIRPS data @@ -162,7 +166,6 @@ def Download(self, progress_bar: bool=True, cores=None): ) return results - @staticmethod def RetrieveData(Date, args): """RetrieveData. @@ -198,8 +201,8 @@ def RetrieveData(Date, args): # Define FTP path to directory if TimeCase == "daily": pathFTP = ( - "pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/" - % Date.strftime("%Y") + "pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/" + % Date.strftime("%Y") ) elif TimeCase == "monthly": pathFTP = "pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/" @@ -263,7 +266,7 @@ def RetrieveData(Date, args): dataset, NoDataValue = Raster.getRasterData(src) # clip dataset to the given extent - data = dataset[yID[0]: yID[1], xID[0]: xID[1]] + data = dataset[yID[0] : yID[1], xID[0] : xID[1]] # replace -ve values with -9999 data[data < 0] = -9999 @@ -281,14 +284,13 @@ def RetrieveData(Date, args): os.remove(outfilename) except PermissionError: - print("The file covering the whole world could not be deleted please delete it after the download ends") + print( + "The file covering the whole world could not be deleted please delete it after the download ends" + ) return True - def ListAttributes(self): - """ - Print Attributes List - """ + """Print Attributes List.""" print("\n") print( @@ -300,4 +302,4 @@ def ListAttributes(self): if key != "name": print(str(key) + " : " + repr(self.__dict__[key])) - print("\n") \ No newline at end of file + print("\n") diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index b59da02..5f4cbe3 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -1,11 +1,11 @@ -""" -Created on Fri Apr 2 06:58:20 2021 +"""Created on Fri Apr 2 06:58:20 2021. @author: mofarrag """ import calendar import datetime as dt import os + import numpy as np import pandas as pd from ecmwfapi import ECMWFDataServer @@ -14,6 +14,7 @@ from earth2observe.utils import print_progress_bar + class ECMWF: """RemoteSensing. @@ -25,18 +26,19 @@ class ECMWF: 3- API 4- ListAttributes """ + def __init__( - self, - time: str = "daily", - start: str = "", - end: str = "", - path: str = "", - variables: list=[], - lat_lim: list=[], - lon_lim: list=[], - fmt: str="%Y-%m-%d", + self, + time: str = "daily", + start: str = "", + end: str = "", + path: str = "", + variables: list = [], + lat_lim: list = [], + lon_lim: list = [], + fmt: str = "%Y-%m-%d", ): - """RemoteSensing + """RemoteSensing. Parameters ---------- @@ -86,9 +88,8 @@ def __init__( # for ECMWF only self.string7 = "%s/to/%s" % (self.start, self.end) - def download(self, progress_bar: bool = True): - """ECMWF + """ECMWF. ECMWF method downloads ECMWF daily data for a given variable, time interval, and spatial extent. @@ -105,17 +106,17 @@ def download(self, progress_bar: bool = True): """ for var in self.vars: # Download data - print(f"\nDownload ECMWF {var} data for period {self.start} till {self.end}") + print( + f"\nDownload ECMWF {var} data for period {self.start} till {self.end}" + ) self.downloadData(var, progress_bar) # CaseParameters=[SumMean, Min, Max] # delete the downloaded netcdf del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc") os.remove(del_ecmwf_dataset) - def downloadData(self, var: str, progress_bar: bool): - """ - This function downloads ECMWF six-hourly, daily or monthly data + """This function downloads ECMWF six-hourly, daily or monthly data. Parameters ---------- @@ -280,21 +281,20 @@ def downloadData(self, var: str, progress_bar: bool): return () - @staticmethod def API( - output_folder, - DownloadType, - string1, - string2, - string3, - string4, - string5, - string6, - string7, - string8, - string9, - string10, + output_folder, + DownloadType, + string1, + string2, + string3, + string4, + string5, + string6, + string7, + string8, + string9, + string10, ): os.chdir(output_folder) @@ -314,9 +314,9 @@ def API( "time": "%s" % string6, "date": "%s" % string7, "type": "%s" - % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ + % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ "class": "%s" - % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ + % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": "%s" % string10, "format": "netcdf", "target": "data_interim.nc", @@ -336,9 +336,9 @@ def API( "time": "%s" % string6, "date": "%s" % string7, "type": "%s" - % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ + % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/ "class": "%s" - % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ + % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": "%s" % string10, "format": "netcdf", "target": "data_interim.nc", @@ -349,10 +349,8 @@ def API( class Variables: - """ - This class contains the information about the ECMWF variables - http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc - """ + """This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc.""" + number_para = { "T": 130, "2T": 167, @@ -540,7 +538,6 @@ class Variables: "HCC": 1, } - def __init__(self, step): # output units after applying factor @@ -619,17 +616,13 @@ def __init__(self, step): else: raise KeyError("The input time step is not supported") - def __str__(self): print( f"Variable name:\n {self.var_name}\nDescriptions\n{self.descriptions}\nUnits : \n{self.units}" ) - def ListAttributes(self): - """ - Print Attributes List - """ + """Print Attributes List.""" print("\n") print( f"Attributes List of: {repr(self.__dict__['name'])} - {self.__class__.__name__} Instance\n" diff --git a/earth2observe/gee/__init__.py b/earth2observe/gee/__init__.py index 3bc65af..223ca29 100644 --- a/earth2observe/gee/__init__.py +++ b/earth2observe/gee/__init__.py @@ -13,7 +13,7 @@ # documentation format __author__ = "Mostafa Farrag" -__email__ = 'moah.farag@gmail.com' +__email__ = "moah.farag@gmail.com" __docformat__ = "restructuredtext" # Let users know if they're missing any of our hard dependencies @@ -25,13 +25,16 @@ __import__(dependency) except ImportError as e: missing_dependencies.append(dependency) + print(e) if missing_dependencies: raise ImportError("Missing required dependencies {0}".format(missing_dependencies)) -import earth2observe.gee.data as data -import earth2observe.gee.dataset as dataset -import earth2observe.gee.gee as gee +# import earth2observe.gee.data as data +# import earth2observe.gee.dataset as dataset +# import earth2observe.gee.features as features +# import earth2observe.gee.gee as gee +# import earth2observe.gee.images as images __doc__ = """ gee - google earth engine diff --git a/earth2observe/gee/dataset.py b/earth2observe/gee/dataset.py index d2b02e4..0d021fa 100644 --- a/earth2observe/gee/dataset.py +++ b/earth2observe/gee/dataset.py @@ -1,37 +1,46 @@ import datetime as dt -import ee +# import ee +from geopandas.geodataframe import GeoDataFrame from earth2observe.gee.data import getCatalog +from earth2observe.gee.gee import GEE catalog = getCatalog() default_date_format = "%Y-%m-%d" -class Dataset: - """ - Dataset - """ - def __init__(self, dataset_id: str, start_date: str, end_date: str, date_format: str = "%Y-%m-%d"): + +class Dataset(GEE): + """Dataset.""" + + def __init__( + self, + dataset_id: str, + start_date: str, + end_date: str, + date_format: str = "%Y-%m-%d", + ): if dataset_id not in catalog["dataset"].tolist(): - raise ValueError(f"the given dataset: {dataset_id} does nor exist in the catalog") + raise ValueError( + f"the given dataset: {dataset_id} does nor exist in the catalog" + ) else: self.metadata = catalog.loc[catalog["dataset"] == dataset_id, :] self.id = id - self.start_date, self.end_date = self.getDate(dataset_id, start_date, end_date, date_format) - self.catalog = catalog - pass - - - - + self.start_date, self.end_date = self.getDate( + dataset_id, start_date, end_date, date_format + ) + # self.catalog = catalog + self.boundary = None @staticmethod def getDate( - dataset_id: str, - start_date: str = None, - end_date: str = None, - date_format: str = default_date_format): + dataset_id: str, + start_date: str = None, + end_date: str = None, + date_format: str = default_date_format, + ): """getDate. getDate retrieves the start and end date of a dataset @@ -57,7 +66,9 @@ def getDate( """ data = catalog.loc[catalog["dataset"] == dataset_id, :] - dataset_start_date = dt.datetime.strptime(data["start_date"].values[0], default_date_format) + dataset_start_date = dt.datetime.strptime( + data["start_date"].values[0], default_date_format + ) dataset_end_date = data["end_date"].values[0] if dataset_end_date == "Now": dataset_end_date = dt.datetime.now().date() @@ -77,3 +88,26 @@ def getDate( end_date = dataset_end_date return start_date, end_date + + def addBoundary(self, gdf: GeoDataFrame): + """addBoundary. + + addBoundary + + Parameters + ---------- + gdf + """ + self.boundary = gdf.copy() + + def filterByRegion(self, gdf: GeoDataFrame = None): + """filterByRegion. + + filterByRegion + + Parameters + ---------- + gdf + """ + if gdf: + self.addBoundary(gdf) diff --git a/earth2observe/gee/features.py b/earth2observe/gee/features.py new file mode 100644 index 0000000..a331b1e --- /dev/null +++ b/earth2observe/gee/features.py @@ -0,0 +1,95 @@ +from typing import List, Optional, Union + +import ee +import pandas as pd +from ee.featurecollection import FeatureCollection +from ee.geometry import Geometry +from geopandas.geodataframe import GeoDataFrame +from loguru import logger +from shapely.geometry import LineString, Point, Polygon + + +def createGeometry( + shapely_geometry: Union[Polygon, Point, LineString], + epsg: int = 4326, +) -> Geometry: + """createGeometry. + + create earth engine geometry. + + Parameters + ---------- + shapely_geometry: [shapely.geometry] + shapely geometry object [point, polyline, Linestring] + epsg: [int] + projection epsg number. + + Returns + ------- + ee Geometry : + ee geometry object [Polygon, Point, LineString] + """ + coords = shapely_geometry.__geo_interface__["coordinates"] + if shapely_geometry.geom_type == "Polygon": + return ee.Geometry.Polygon(coords, f"epsg:{epsg}") + + elif shapely_geometry.geom_type in ["Point", "LineString"]: + if shapely_geometry.geom_type == "LineString": + raise ValueError("LineStrings not yet implemented.") + else: + return ee.Geometry.Point(coords, f"epsg:{epsg}") + + else: + logger.debug( + f"The given geometry is neiter of type LineString, Point nor Polygon, " + f"but {shapely_geometry.geom_type}." + ) + + +def createFeature( + gdf: GeoDataFrame, columns: Optional[List[str]] = None +) -> FeatureCollection: + """createFeature. + + createFeature creates a feature collection from a geodataframe + + collection with the data in a certain column in the geodataframe as a properties dictionary + + Parameters + ---------- + gdf : [GeoDataFrame] + + columns: [list] + list of strings for the columns' names + + Returns + ------- + FeatureCollection : [ee.featurecollection.FeatureCollection] + feature collection containing the geometry of each row in the given geodataframe + with the information of one of the given columns as a property. + """ + try: + # get the geometry type for all rows + geotype = [i.geom_type for i in gdf["geometry"]] + # if any is "MultiPolygon" explode the dataframe to single polygons + if "MultiPolygon" in geotype: + # index_parts=True makes the resulted index multi-index if multi-polygon resulted in + # many different polygons + gdf = gdf.explode(index_parts=True) + + ee_geom_list = gdf.geometry.apply(lambda geom: createGeometry(geom)).to_list() + records_df = pd.DataFrame(gdf.drop("geometry", axis=1)) + if columns: + records_df = records_df[columns] + records = records_df.to_dict("records") + if not records: + ee_feature_list = [ee.Feature(geom) for geom in ee_geom_list] + else: + ee_feature_list = [ + ee.Feature(geom, record) for geom, record in zip(ee_geom_list, records) + ] + return ee.FeatureCollection(ee_feature_list) + + except Exception as error: + logger.error(error) + raise ValueError(error) diff --git a/earth2observe/gee/gee.py b/earth2observe/gee/gee.py index c11d592..9a4ccbe 100644 --- a/earth2observe/gee/gee.py +++ b/earth2observe/gee/gee.py @@ -1,16 +1,16 @@ -"Google earth engine main script" +"""Google earth engine main script.""" import base64 import json -import os import ee +# import os + class GEE: - """ - GEE - """ - def __init__(self, service_account:str, service_key_path: str): + """GEE.""" + + def __init__(self, service_account: str, service_key_path: str): """Initialize. Parameters @@ -23,11 +23,12 @@ def __init__(self, service_account:str, service_key_path: str): ------- None """ - self.Initialize(service_account, service_key_path) + self.initialize(service_account, service_key_path) pass - def Initialize(self, service_account: str, service_key: str): + @staticmethod + def initialize(service_account: str, service_key: str): """Initialize. Initialize authenticate and initializes the connection to google earth engine with a service accont file @@ -47,10 +48,11 @@ def Initialize(self, service_account: str, service_key: str): try: credentials = ee.ServiceAccountCredentials(service_account, service_key) except ValueError: - credentials = ee.ServiceAccountCredentials(service_account, key_data=service_key) + credentials = ee.ServiceAccountCredentials( + service_account, key_data=service_key + ) ee.Initialize(credentials=credentials) - @staticmethod def encodeServiceAccount(service_key_dir: str) -> bytes: """encodeServiceAccount. @@ -70,7 +72,6 @@ def encodeServiceAccount(service_key_dir: str) -> bytes: encoded_service_account = base64.b64encode(dumped_service_account.encode()) return encoded_service_account - @staticmethod def decodeServiceAccount(service_key_bytes: bytes) -> str: """decodeServiceAccount. diff --git a/earth2observe/gee/imagecollection.py b/earth2observe/gee/imagecollection.py index a4a00d0..a386470 100644 --- a/earth2observe/gee/imagecollection.py +++ b/earth2observe/gee/imagecollection.py @@ -1,2 +1 @@ - # collections = ee.ImageCollection(url') diff --git a/earth2observe/gee/images.py b/earth2observe/gee/images.py new file mode 100644 index 0000000..e69de29 diff --git a/earth2observe/utils.py b/earth2observe/utils.py index 1ab00e5..9f57fec 100644 --- a/earth2observe/utils.py +++ b/earth2observe/utils.py @@ -1,15 +1,16 @@ -import sys -import os import gzip +import os +import sys + def print_progress_bar( - i: int, - total: int, - prefix: str="", - suffix: str="", - decimals: int=1, - length: int=100, - fill: str="█" + i: int, + total: int, + prefix: str = "", + suffix: str = "", + decimals: int = 1, + length: int = 100, + fill: str = "█", ): """print_progress_bar. @@ -45,11 +46,8 @@ def print_progress_bar( print() - def extractFromGZ(input_file, output_file, delete=False): - """ - ExtractFromGZ method extract data from the zip/.gz files, - save the data + """ExtractFromGZ method extract data from the zip/.gz files, save the data. Parameters ---------- @@ -73,4 +71,4 @@ def extractFromGZ(input_file, output_file, delete=False): zf.close() if delete: - os.remove(input_file) \ No newline at end of file + os.remove(input_file) diff --git a/environment.yml b/environment.yml index 17d354c..8cd6d58 100644 --- a/environment.yml +++ b/environment.yml @@ -1,16 +1,16 @@ channels: - conda-forge dependencies: - - python >=3.7.1,<3.10 - - pip >=21.3.1 - - numpy >=1.21.2,<1.22.4 - - netCDF4 >=1.5.5,<1.5.9 + - python >=3.9,<3.10 + - pip >=22.2.2 + - numpy >=1.21.2,<1.23.3 + - netCDF4 >=1.5.5,<1.6.1 - gdal >=3.3.3,<3.5.1 - - pandas >=1.3.2,<1.4.3 - - pyramids >=0.1.0 - - ecmwf-api-client >=1.6.1 - - earthengine-api + - pandas >=1.3.2,<1.5.0 + - pyramids >=0.2.2 + - ecmwf-api-client >=1.6.3 + - earthengine-api >=0.1.324 - pip: - - loguru >=0.5.3 - - pytest >=6.2.5 - - pytest-cov >=2.12.1 + - loguru >=0.6.0 + - pytest >=7.1.3 + - pytest-cov >=3.0.0 diff --git a/examples/Download Satellite data.ipynb b/examples/Download Satellite data.ipynb index cc6c291..2448ba0 100644 --- a/examples/Download Satellite data.ipynb +++ b/examples/Download Satellite data.ipynb @@ -488,4 +488,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/examples/Download Satellite data.py b/examples/Download Satellite data.py index ea55d01..a6f42ed 100644 --- a/examples/Download Satellite data.py +++ b/examples/Download Satellite data.py @@ -1,14 +1,12 @@ -""" -Download Satellite data -ECMWF -Installation of ECMWF API key +"""Download Satellite data ECMWF Installation of ECMWF API key. + 1 - to be able to use Hapi to download ECMWF data you need to register and setup your account in the ECMWF website (https://apps.ecmwf.int/registration/) 2 - Install ECMWF key (instruction are here https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key) """ from earth2observe.chirps import CHIRPS -from earth2observe.ecmwf import ECMWF -from earth2observe.ecmwf import Variables +from earth2observe.ecmwf import ECMWF, Variables + #%% precipitation start = "2009-01-01" end = "2009-01-10" diff --git a/examples/data/ecmwf/data_interim.nc.aux.xml b/examples/data/ecmwf/data_interim.nc.aux.xml new file mode 100644 index 0000000..f52faee --- /dev/null +++ b/examples/data/ecmwf/data_interim.nc.aux.xml @@ -0,0 +1,56 @@ + + + + + 2927.024999999818 + 11951.97500000002 + 60 + 0 + 0 + 1|0|0|0|2|1|2|0|0|0|0|0|0|0|0|0|0|1|0|0|1|0|0|1|1|1|0|2|1|2|2|2|2|0|0|0|0|2|3|2|3|1|1|0|2|4|1|1|3|3|1|3|1|1|2|1|1|1|0|1 + + + + 11878 + 8578.0333333333 + 3001 + 2229.0031999877 + + + + + + -15031.54999999978 + -4594.449999999837 + 60 + 0 + 0 + 1|0|1|2|2|1|3|2|2|2|2|0|2|1|0|3|1|0|2|0|3|0|1|1|2|2|0|0|0|0|0|0|2|3|1|0|0|0|0|0|0|0|5|1|0|0|0|0|0|0|4|2|0|2|1|0|1|0|1|1 + + + + -4680 + -10509.45 + -14946 + 3127.8967929745 + + + + + + -24705.23333333354 + -4302.766666666692 + 60 + 0 + 0 + 1|0|1|1|0|2|1|1|3|1|1|3|1|2|1|1|2|0|1|2|1|2|0|0|2|2|0|2|0|1|1|2|0|1|1|0|1|2|2|0|2|1|0|1|2|1|1|0|1|1|0|1|0|2|0|0|0|1|0|1 + + + + -4470 + -15696.383333333 + -24538 + 5399.3462693542 + + + diff --git a/examples/data/linestring.geojson b/examples/data/linestring.geojson new file mode 100644 index 0000000..3878729 --- /dev/null +++ b/examples/data/linestring.geojson @@ -0,0 +1,74 @@ +{ + "type": "FeatureCollection", + "name": "linestring", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + } + }, + "features": [ + { + "type": "Feature", + "properties": { + "fid": 1 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + -75.02773539668702, + 4.472264603312993 + ], + [ + -74.89270924149957, + 4.231963818657371 + ], + [ + -75.24972755013079, + 4.502016129032262 + ], + [ + -75.21997602441152, + 4.334949869224066 + ], + [ + -75.3000762859634, + 4.126689189189192 + ], + [ + -75.53808849171753, + 4.392164341761119 + ] + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 2 + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + -74.60663687881431, + 4.691968177855278 + ], + [ + -74.54484524847429, + 4.504304707933743 + ], + [ + -74.83320619006103, + 4.6507737576286 + ], + [ + -74.71648866608545, + 4.366989973844816 + ] + ] + } + } + ] +} diff --git a/examples/data/point.gpkg b/examples/data/point.gpkg new file mode 100644 index 0000000..b26f330 Binary files /dev/null and b/examples/data/point.gpkg differ diff --git a/examples/data/points.geojson b/examples/data/points.geojson new file mode 100644 index 0000000..da114e5 --- /dev/null +++ b/examples/data/points.geojson @@ -0,0 +1,90 @@ +{ + "type": "FeatureCollection", + "name": "points", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + } + }, + "features": [ + { + "type": "Feature", + "properties": { + "fid": 1 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -75.24972755013079, + 4.50201612903226 + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 2 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -75.02544681778554, + 4.472264603312993 + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 3 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -75.21997602441152, + 4.330372711421102 + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 4 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -74.89499782040106, + 4.234252397558852 + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 5 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -75.30694202266784, + 4.126689189189192 + ] + } + }, + { + "type": "Feature", + "properties": { + "fid": 6 + }, + "geometry": { + "type": "Point", + "coordinates": [ + -75.53579991281606, + 4.392164341761119 + ] + } + } + ] +} diff --git a/examples/data/points.qmd b/examples/data/points.qmd new file mode 100644 index 0000000..872fba3 --- /dev/null +++ b/examples/data/points.qmd @@ -0,0 +1,26 @@ + + + + + + dataset + + + + + + + + + + 0 + 0 + + + + + false + + + + diff --git a/examples/data/polygon.geojson b/examples/data/polygon.geojson new file mode 100644 index 0000000..89cee2d --- /dev/null +++ b/examples/data/polygon.geojson @@ -0,0 +1,45 @@ +{ + "type": "FeatureCollection", + "name": "polygon", + "crs": { + "type": "name", + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + } + }, + "features": [ + { + "type": "Feature", + "properties": { + "fid": 1 + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -75.74692131647775, + 4.749182650392328 + ], + [ + -74.49850152571925, + 4.750326939843069 + ], + [ + -74.49964581517, + 4.000817349607672 + ], + [ + -75.75035418482997, + 4.000817349607672 + ], + [ + -75.74692131647775, + 4.749182650392328 + ] + ] + ] + } + } + ] +} diff --git a/setup.py b/setup.py index c5e4cb7..75cd8a0 100644 --- a/setup.py +++ b/setup.py @@ -3,27 +3,29 @@ with open("README.md", "r") as readme_file: readme = readme_file.read() -with open('HISTORY.rst') as history_file: +with open("HISTORY.rst") as history_file: history = history_file.read() # requirements = [line.strip() for line in open("requirements.txt").readlines()] # requirements = requirements[1:] -test_requirements = ['pytest>=3', ] +test_requirements = [ + "pytest>=3", +] setup( name="earth2observe", - version="0.1.0", + version="0.1.1", description="remote sensing package", author="Mostafa Farrag", author_email="moah.farag@gmail.come", url="https://github.com/MAfarrag/earthobserve", keywords=["remote sensing", "ecmwf"], - long_description=readme + '\n\n' + history, + long_description=readme + "\n\n" + history, long_description_content_type="text/markdown", license="GNU General Public License v3", zip_safe=False, - packages=find_packages(include=['earth2observe', 'earth2observe.*']), + packages=find_packages(include=["earth2observe", "earth2observe.*"]), test_suite="tests", tests_require=test_requirements, # install_requires=requirements, @@ -35,9 +37,9 @@ classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Natural Language :: English", - 'Programming Language :: Python :: 3', + "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/tests/conftest.py b/tests/conftest.py index 1c03837..7369fbd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,6 @@ # from typing import List - # @pytest.fixture(scope="module") # def time_series1() -> list: - # return pd.read_csv("examples/data/time_series1.txt", header=None)[0].tolist() +# return pd.read_csv("examples/data/time_series1.txt", header=None)[0].tolist() diff --git a/tests/gee/conftest.py b/tests/gee/conftest.py index dad867c..af8bff8 100644 --- a/tests/gee/conftest.py +++ b/tests/gee/conftest.py @@ -5,5 +5,17 @@ @pytest.fixture(scope="module") def catalog_columns() -> List[str]: - return ['dataset', 'name', 'provider', 'url', 'bands', 'band_describtion', 'spatial_resolution', - 'temporal_resolution', 'start_date', 'end_date', 'min', 'max'] + return [ + "dataset", + "name", + "provider", + "url", + "bands", + "band_describtion", + "spatial_resolution", + "temporal_resolution", + "start_date", + "end_date", + "min", + "max", + ]