diff --git a/.coveralls.yml b/.coveralls.yml
index 84b054c..06e982e 100644
--- a/.coveralls.yml
+++ b/.coveralls.yml
@@ -1,3 +1,2 @@
repo_token: hxJrvjqiH2xBI7eit7BAb7FidH0LeYpGq
service_name: github-action
-
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 9f24714..eda6933 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -48,11 +48,11 @@ jobs:
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
-
+
# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality
-
+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
@@ -61,7 +61,7 @@ jobs:
# âšī¸ Command-line programs to run using the OS shell.
# đ See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
- # If the Autobuild fails above, remove it and uncomment the following three lines.
+ # If the Autobuild fails above, remove it and uncomment the following three lines.
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
# - run: |
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
index d19bf87..8f19042 100644
--- a/.github/workflows/workflow.yml
+++ b/.github/workflows/workflow.yml
@@ -62,4 +62,4 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage1.xml,./coverage2.xml
- directory: ./coverage/reports/
\ No newline at end of file
+ directory: ./coverage/reports/
diff --git a/.gitignore b/.gitignore
index f65edba..dc7684d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -147,3 +147,6 @@ build_artifacts
mo_*
conda/
/examples/data/ecmwf/daily/*
+.idea/*
+poetry.lock
+pyproject.toml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5450999..e3bc9b5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,14 +1,12 @@
repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v2.3.0
- hooks:
-# - id: check-yaml
-# name: "[yaml - check] validate yaml"
- - id: end-of-file-fixer
- name: "[py - check] validate yaml"
- - id: trailing-whitespace
- name: "[file - format] trim trailing whitespace"
- args: [ --markdown-linebreak-ext=md ]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.3.0
+ hooks:
+ - id: end-of-file-fixer
+ name: "[py - check] validate yaml"
+ - id: trailing-whitespace
+ name: "[file - format] trim trailing whitespace"
+ args: [ --markdown-linebreak-ext=md ]
- id: check-added-large-files
name: "[file - check] large file"
args: [ --maxkb=5000 ]
@@ -37,41 +35,51 @@ repos:
--no-sort-keys ]
- id: requirements-txt-fixer
name: "[reqs - format] fix requirements.txt"
+ - id: check-yaml
+ name: "[yaml - check] validate yaml"
+- repo: https://github.com/PyCQA/docformatter
+ rev: v1.4
+ hooks:
+ - id: docformatter
+ name: "[py - format] docformatter"
+ args: [ -i, --wrap-summaries, "0" ]
-# - repo: https://github.com/PyCQA/docformatter
-# rev: v1.4
-# hooks:
-# - id: docformatter
-## name: "[py - format] docformatter"
-## args: [ -i, --wrap-summaries, "0" ]
+- repo: https://github.com/PyCQA/pydocstyle
+ rev: 6.1.1
+ hooks:
+ - id: pydocstyle
+ name: "[py - check] pydocstyle"
+ files: ^Hapi/
-# - repo: https://github.com/PyCQA/pydocstyle
-# rev: 6.1.1
-# hooks:
-# - id: pydocstyle
-# name: "[py - check] pydocstyle"
-# files: ^Hapi/
- # TODO : uncheck later and fix all the problems of line too long
-#- repo: https://gitlab.com/pycqa/flake8
-# rev: 3.8.4
-# hooks:
-# - id: flake8
-# name: "[py - check] flake8"
-# language_version: python3.8
-# TODO : this hook does not fix the files
- #- repo: https://github.com/psf/black
-# rev: 19.3b0
-# hooks:
-# - id: black
-- repo: https://github.com/pre-commit/mirrors-isort
- rev: v5.7.0
- hooks:
- - id: isort
- name: "[py - format] isort"
+- repo: https://gitlab.com/pycqa/flake8
+ rev: 3.8.4
+ hooks:
+ - id: flake8
+ name: "[py - check] flake8"
+ language_version: python3.9
+ exclude: ^(examples/|tests/)
-#- repo: https://github.com/ambv/black
-# rev: 20.8b1
-# hooks:
-# - id: black
-# name: "[py - format] black"
-# language_version: python3.8
+- repo: https://github.com/psf/black
+ rev: 22.8.0
+ hooks:
+ - id: black
+- repo: https://github.com/pre-commit/mirrors-isort
+ rev: v5.7.0
+ hooks:
+ - id: isort
+ name: "[py - format] isort"
+- repo: https://github.com/ambv/black
+ rev: 22.8.0
+ hooks:
+ - id: black
+ name: "[py - format] black"
+ language_version: python3.9
+
+- repo: local
+ hooks:
+ - id: pytest-check
+ name: pytest-check
+ entry: pytest
+ language: system
+ pass_filenames: false
+ always_run: true
diff --git a/README.md b/README.md
index 05262a8..0799257 100644
--- a/README.md
+++ b/README.md
@@ -25,12 +25,12 @@ earthobserve
Main Features
-------------
- -
+ -
Future work
-------------
- -
+ -
diff --git a/clone.json b/clone.json
index ed40d55..84f855d 100644
--- a/clone.json
+++ b/clone.json
@@ -1,4 +1,4 @@
{
- "message": "Must have push access to repository",
- "documentation_url": "https://docs.github.com/rest/reference/repos#get-repository-clones"
+ "message": "Must have push access to repository",
+ "documentation_url": "https://docs.github.com/rest/reference/repos#get-repository-clones"
}
diff --git a/docs/conf.py b/docs/conf.py
index bb14dd7..11e7682 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -9,18 +9,17 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-# General information about the project.
-project = u"pyramids"
-author = "Mostafa Farrag"
-
-# copyright = u"2013-2019, "
-
-
import os
import sys
# import sphinx_rtd_theme
+# General information about the project.
+project = "pyramids"
+author = "Mostafa Farrag"
+
+# copyright = u"2013-2019, "
+
html_theme = "sphinxdoc"
# html_theme = "agogo"
html_theme_path = ["."]
@@ -207,7 +206,7 @@
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
- ("index", "pyramids.tex", u"pyramids Documentation", u"Mostafa Farrag", "report")
+ ("index", "pyramids.tex", "pyramids Documentation", "Mostafa Farrag", "report")
]
# The name of an image file (relative to this directory) to place at the top of
@@ -250,8 +249,8 @@
(
"index",
"pyramids",
- u"pyramids Documentation",
- u"Mostafa Farrag",
+ "pyramids Documentation",
+ "Mostafa Farrag",
"pyramids",
"One line description of project.",
"Miscellaneous",
diff --git a/earth2observe/__init__.py b/earth2observe/__init__.py
index 6864b85..a47cd93 100644
--- a/earth2observe/__init__.py
+++ b/earth2observe/__init__.py
@@ -13,7 +13,7 @@
# documentation format
__author__ = "Mostafa Farrag"
-__email__ = 'moah.farag@gmail.com'
+__email__ = "moah.farag@gmail.com"
__docformat__ = "restructuredtext"
# Let users know if they're missing any of our hard dependencies
@@ -25,15 +25,17 @@
__import__(dependency)
except ImportError as e:
missing_dependencies.append(dependency)
+ print(e)
if missing_dependencies:
raise ImportError("Missing required dependencies {0}".format(missing_dependencies))
-def configuration(parent_package='',top_path=None):
+
+def configuration(parent_package="", top_path=None):
from numpy.distutils.misc_util import Configuration
- config = Configuration(None,parent_package,top_path)
+ config = Configuration(None, parent_package, top_path)
config.set_options(
ignore_setup_xxx_py=True,
assume_default_configuration=True,
@@ -41,14 +43,14 @@ def configuration(parent_package='',top_path=None):
quiet=True,
)
- config.add_subpackage('gee')
+ config.add_subpackage("gee")
return config
-import earth2observe.chirps as chirps
-import earth2observe.ecmwf as ecmwf
-import earth2observe.gee as gee
-import earth2observe.utils as utils
+# import earth2observe.chirps as chirps
+# import earth2observe.ecmwf as ecmwf
+# import earth2observe.gee as gee
+# import earth2observe.utils as utils
__doc__ = """
earth2observe - remote sensing package
diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py
index d0eb191..56d87a7 100644
--- a/earth2observe/chirps.py
+++ b/earth2observe/chirps.py
@@ -1,27 +1,28 @@
-import os
import datetime as dt
+import os
+from ftplib import FTP
+
import numpy as np
import pandas as pd
-from ftplib import FTP
from joblib import Parallel, delayed
from osgeo import gdal
from pyramids.raster import Raster
-from earth2observe.utils import print_progress_bar, extractFromGZ
+
+from earth2observe.utils import extractFromGZ, print_progress_bar
class CHIRPS:
- """
- CHIRPS
- """
+ """CHIRPS."""
+
def __init__(
- self,
- start: str="",
- end: str="",
- lat_lim: list=[],
- lon_lim: list=[],
- time: str="daily",
- path: str="",
- fmt: str="%Y-%m-%d",
+ self,
+ start: str = "",
+ end: str = "",
+ lat_lim: list = [],
+ lon_lim: list = [],
+ time: str = "daily",
+ path: str = "",
+ fmt: str = "%Y-%m-%d",
):
"""CHIRPS.
@@ -52,7 +53,9 @@ def __init__(
self.output_folder = os.path.join(path, "Precipitation", "CHIRPS", "Daily")
elif time == "monthly":
self.time_freq = "MS"
- self.output_folder = os.path.join(path, "Precipitation", "CHIRPS", "Monthly")
+ self.output_folder = os.path.join(
+ path, "Precipitation", "CHIRPS", "Monthly"
+ )
else:
raise KeyError("The input time interval is not supported")
self.time = time
@@ -94,7 +97,9 @@ def __init__(
self.lon_lim = lon_lim
# Define IDs
self.yID = 2000 - np.int16(
- np.array([np.ceil((lat_lim[1] + 50) * 20), np.floor((lat_lim[0] + 50) * 20)])
+ np.array(
+ [np.ceil((lat_lim[1] + 50) * 20), np.floor((lat_lim[0] + 50) * 20)]
+ )
)
self.xID = np.int16(
np.array(
@@ -102,8 +107,7 @@ def __init__(
)
)
-
- def Download(self, progress_bar: bool=True, cores=None):
+ def Download(self, progress_bar: bool = True, cores=None):
"""Download.
Download method downloads CHIRPS data
@@ -162,7 +166,6 @@ def Download(self, progress_bar: bool=True, cores=None):
)
return results
-
@staticmethod
def RetrieveData(Date, args):
"""RetrieveData.
@@ -198,8 +201,8 @@ def RetrieveData(Date, args):
# Define FTP path to directory
if TimeCase == "daily":
pathFTP = (
- "pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/"
- % Date.strftime("%Y")
+ "pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p05/%s/"
+ % Date.strftime("%Y")
)
elif TimeCase == "monthly":
pathFTP = "pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/"
@@ -263,7 +266,7 @@ def RetrieveData(Date, args):
dataset, NoDataValue = Raster.getRasterData(src)
# clip dataset to the given extent
- data = dataset[yID[0]: yID[1], xID[0]: xID[1]]
+ data = dataset[yID[0] : yID[1], xID[0] : xID[1]]
# replace -ve values with -9999
data[data < 0] = -9999
@@ -281,14 +284,13 @@ def RetrieveData(Date, args):
os.remove(outfilename)
except PermissionError:
- print("The file covering the whole world could not be deleted please delete it after the download ends")
+ print(
+ "The file covering the whole world could not be deleted please delete it after the download ends"
+ )
return True
-
def ListAttributes(self):
- """
- Print Attributes List
- """
+ """Print Attributes List."""
print("\n")
print(
@@ -300,4 +302,4 @@ def ListAttributes(self):
if key != "name":
print(str(key) + " : " + repr(self.__dict__[key]))
- print("\n")
\ No newline at end of file
+ print("\n")
diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py
index b59da02..5f4cbe3 100644
--- a/earth2observe/ecmwf.py
+++ b/earth2observe/ecmwf.py
@@ -1,11 +1,11 @@
-"""
-Created on Fri Apr 2 06:58:20 2021
+"""Created on Fri Apr 2 06:58:20 2021.
@author: mofarrag
"""
import calendar
import datetime as dt
import os
+
import numpy as np
import pandas as pd
from ecmwfapi import ECMWFDataServer
@@ -14,6 +14,7 @@
from earth2observe.utils import print_progress_bar
+
class ECMWF:
"""RemoteSensing.
@@ -25,18 +26,19 @@ class ECMWF:
3- API
4- ListAttributes
"""
+
def __init__(
- self,
- time: str = "daily",
- start: str = "",
- end: str = "",
- path: str = "",
- variables: list=[],
- lat_lim: list=[],
- lon_lim: list=[],
- fmt: str="%Y-%m-%d",
+ self,
+ time: str = "daily",
+ start: str = "",
+ end: str = "",
+ path: str = "",
+ variables: list = [],
+ lat_lim: list = [],
+ lon_lim: list = [],
+ fmt: str = "%Y-%m-%d",
):
- """RemoteSensing
+ """RemoteSensing.
Parameters
----------
@@ -86,9 +88,8 @@ def __init__(
# for ECMWF only
self.string7 = "%s/to/%s" % (self.start, self.end)
-
def download(self, progress_bar: bool = True):
- """ECMWF
+ """ECMWF.
ECMWF method downloads ECMWF daily data for a given variable, time
interval, and spatial extent.
@@ -105,17 +106,17 @@ def download(self, progress_bar: bool = True):
"""
for var in self.vars:
# Download data
- print(f"\nDownload ECMWF {var} data for period {self.start} till {self.end}")
+ print(
+ f"\nDownload ECMWF {var} data for period {self.start} till {self.end}"
+ )
self.downloadData(var, progress_bar) # CaseParameters=[SumMean, Min, Max]
# delete the downloaded netcdf
del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc")
os.remove(del_ecmwf_dataset)
-
def downloadData(self, var: str, progress_bar: bool):
- """
- This function downloads ECMWF six-hourly, daily or monthly data
+ """This function downloads ECMWF six-hourly, daily or monthly data.
Parameters
----------
@@ -280,21 +281,20 @@ def downloadData(self, var: str, progress_bar: bool):
return ()
-
@staticmethod
def API(
- output_folder,
- DownloadType,
- string1,
- string2,
- string3,
- string4,
- string5,
- string6,
- string7,
- string8,
- string9,
- string10,
+ output_folder,
+ DownloadType,
+ string1,
+ string2,
+ string3,
+ string4,
+ string5,
+ string6,
+ string7,
+ string8,
+ string9,
+ string10,
):
os.chdir(output_folder)
@@ -314,9 +314,9 @@ def API(
"time": "%s" % string6,
"date": "%s" % string7,
"type": "%s"
- % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
+ % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
"class": "%s"
- % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
+ % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
"area": "%s" % string10,
"format": "netcdf",
"target": "data_interim.nc",
@@ -336,9 +336,9 @@ def API(
"time": "%s" % string6,
"date": "%s" % string7,
"type": "%s"
- % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
+ % string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
"class": "%s"
- % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
+ % string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
"area": "%s" % string10,
"format": "netcdf",
"target": "data_interim.nc",
@@ -349,10 +349,8 @@ def API(
class Variables:
- """
- This class contains the information about the ECMWF variables
- http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc
- """
+ """This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc."""
+
number_para = {
"T": 130,
"2T": 167,
@@ -540,7 +538,6 @@ class Variables:
"HCC": 1,
}
-
def __init__(self, step):
# output units after applying factor
@@ -619,17 +616,13 @@ def __init__(self, step):
else:
raise KeyError("The input time step is not supported")
-
def __str__(self):
print(
f"Variable name:\n {self.var_name}\nDescriptions\n{self.descriptions}\nUnits : \n{self.units}"
)
-
def ListAttributes(self):
- """
- Print Attributes List
- """
+ """Print Attributes List."""
print("\n")
print(
f"Attributes List of: {repr(self.__dict__['name'])} - {self.__class__.__name__} Instance\n"
diff --git a/earth2observe/gee/__init__.py b/earth2observe/gee/__init__.py
index 3bc65af..223ca29 100644
--- a/earth2observe/gee/__init__.py
+++ b/earth2observe/gee/__init__.py
@@ -13,7 +13,7 @@
# documentation format
__author__ = "Mostafa Farrag"
-__email__ = 'moah.farag@gmail.com'
+__email__ = "moah.farag@gmail.com"
__docformat__ = "restructuredtext"
# Let users know if they're missing any of our hard dependencies
@@ -25,13 +25,16 @@
__import__(dependency)
except ImportError as e:
missing_dependencies.append(dependency)
+ print(e)
if missing_dependencies:
raise ImportError("Missing required dependencies {0}".format(missing_dependencies))
-import earth2observe.gee.data as data
-import earth2observe.gee.dataset as dataset
-import earth2observe.gee.gee as gee
+# import earth2observe.gee.data as data
+# import earth2observe.gee.dataset as dataset
+# import earth2observe.gee.features as features
+# import earth2observe.gee.gee as gee
+# import earth2observe.gee.images as images
__doc__ = """
gee - google earth engine
diff --git a/earth2observe/gee/dataset.py b/earth2observe/gee/dataset.py
index d2b02e4..0d021fa 100644
--- a/earth2observe/gee/dataset.py
+++ b/earth2observe/gee/dataset.py
@@ -1,37 +1,46 @@
import datetime as dt
-import ee
+# import ee
+from geopandas.geodataframe import GeoDataFrame
from earth2observe.gee.data import getCatalog
+from earth2observe.gee.gee import GEE
catalog = getCatalog()
default_date_format = "%Y-%m-%d"
-class Dataset:
- """
- Dataset
- """
- def __init__(self, dataset_id: str, start_date: str, end_date: str, date_format: str = "%Y-%m-%d"):
+
+class Dataset(GEE):
+ """Dataset."""
+
+ def __init__(
+ self,
+ dataset_id: str,
+ start_date: str,
+ end_date: str,
+ date_format: str = "%Y-%m-%d",
+ ):
if dataset_id not in catalog["dataset"].tolist():
- raise ValueError(f"the given dataset: {dataset_id} does nor exist in the catalog")
+ raise ValueError(
+ f"the given dataset: {dataset_id} does nor exist in the catalog"
+ )
else:
self.metadata = catalog.loc[catalog["dataset"] == dataset_id, :]
self.id = id
- self.start_date, self.end_date = self.getDate(dataset_id, start_date, end_date, date_format)
- self.catalog = catalog
- pass
-
-
-
-
+ self.start_date, self.end_date = self.getDate(
+ dataset_id, start_date, end_date, date_format
+ )
+ # self.catalog = catalog
+ self.boundary = None
@staticmethod
def getDate(
- dataset_id: str,
- start_date: str = None,
- end_date: str = None,
- date_format: str = default_date_format):
+ dataset_id: str,
+ start_date: str = None,
+ end_date: str = None,
+ date_format: str = default_date_format,
+ ):
"""getDate.
getDate retrieves the start and end date of a dataset
@@ -57,7 +66,9 @@ def getDate(
"""
data = catalog.loc[catalog["dataset"] == dataset_id, :]
- dataset_start_date = dt.datetime.strptime(data["start_date"].values[0], default_date_format)
+ dataset_start_date = dt.datetime.strptime(
+ data["start_date"].values[0], default_date_format
+ )
dataset_end_date = data["end_date"].values[0]
if dataset_end_date == "Now":
dataset_end_date = dt.datetime.now().date()
@@ -77,3 +88,26 @@ def getDate(
end_date = dataset_end_date
return start_date, end_date
+
+ def addBoundary(self, gdf: GeoDataFrame):
+ """addBoundary.
+
+ addBoundary
+
+ Parameters
+ ----------
+ gdf
+ """
+ self.boundary = gdf.copy()
+
+ def filterByRegion(self, gdf: GeoDataFrame = None):
+ """filterByRegion.
+
+ filterByRegion
+
+ Parameters
+ ----------
+ gdf
+ """
+ if gdf:
+ self.addBoundary(gdf)
diff --git a/earth2observe/gee/features.py b/earth2observe/gee/features.py
new file mode 100644
index 0000000..a331b1e
--- /dev/null
+++ b/earth2observe/gee/features.py
@@ -0,0 +1,95 @@
+from typing import List, Optional, Union
+
+import ee
+import pandas as pd
+from ee.featurecollection import FeatureCollection
+from ee.geometry import Geometry
+from geopandas.geodataframe import GeoDataFrame
+from loguru import logger
+from shapely.geometry import LineString, Point, Polygon
+
+
+def createGeometry(
+ shapely_geometry: Union[Polygon, Point, LineString],
+ epsg: int = 4326,
+) -> Geometry:
+ """createGeometry.
+
+ create earth engine geometry.
+
+ Parameters
+ ----------
+ shapely_geometry: [shapely.geometry]
+ shapely geometry object [point, polyline, Linestring]
+ epsg: [int]
+ projection epsg number.
+
+ Returns
+ -------
+ ee Geometry :
+ ee geometry object [Polygon, Point, LineString]
+ """
+ coords = shapely_geometry.__geo_interface__["coordinates"]
+ if shapely_geometry.geom_type == "Polygon":
+ return ee.Geometry.Polygon(coords, f"epsg:{epsg}")
+
+ elif shapely_geometry.geom_type in ["Point", "LineString"]:
+ if shapely_geometry.geom_type == "LineString":
+ raise ValueError("LineStrings not yet implemented.")
+ else:
+ return ee.Geometry.Point(coords, f"epsg:{epsg}")
+
+ else:
+ logger.debug(
+ f"The given geometry is neiter of type LineString, Point nor Polygon, "
+ f"but {shapely_geometry.geom_type}."
+ )
+
+
+def createFeature(
+ gdf: GeoDataFrame, columns: Optional[List[str]] = None
+) -> FeatureCollection:
+ """createFeature.
+
+ createFeature creates a feature collection from a geodataframe
+
+ collection with the data in a certain column in the geodataframe as a properties dictionary
+
+ Parameters
+ ----------
+ gdf : [GeoDataFrame]
+
+ columns: [list]
+ list of strings for the columns' names
+
+ Returns
+ -------
+ FeatureCollection : [ee.featurecollection.FeatureCollection]
+ feature collection containing the geometry of each row in the given geodataframe
+ with the information of one of the given columns as a property.
+ """
+ try:
+ # get the geometry type for all rows
+ geotype = [i.geom_type for i in gdf["geometry"]]
+ # if any is "MultiPolygon" explode the dataframe to single polygons
+ if "MultiPolygon" in geotype:
+ # index_parts=True makes the resulted index multi-index if multi-polygon resulted in
+ # many different polygons
+ gdf = gdf.explode(index_parts=True)
+
+ ee_geom_list = gdf.geometry.apply(lambda geom: createGeometry(geom)).to_list()
+ records_df = pd.DataFrame(gdf.drop("geometry", axis=1))
+ if columns:
+ records_df = records_df[columns]
+ records = records_df.to_dict("records")
+ if not records:
+ ee_feature_list = [ee.Feature(geom) for geom in ee_geom_list]
+ else:
+ ee_feature_list = [
+ ee.Feature(geom, record) for geom, record in zip(ee_geom_list, records)
+ ]
+ return ee.FeatureCollection(ee_feature_list)
+
+ except Exception as error:
+ logger.error(error)
+ raise ValueError(error)
diff --git a/earth2observe/gee/gee.py b/earth2observe/gee/gee.py
index c11d592..9a4ccbe 100644
--- a/earth2observe/gee/gee.py
+++ b/earth2observe/gee/gee.py
@@ -1,16 +1,16 @@
-"Google earth engine main script"
+"""Google earth engine main script."""
import base64
import json
-import os
import ee
+# import os
+
class GEE:
- """
- GEE
- """
- def __init__(self, service_account:str, service_key_path: str):
+ """GEE."""
+
+ def __init__(self, service_account: str, service_key_path: str):
"""Initialize.
Parameters
@@ -23,11 +23,12 @@ def __init__(self, service_account:str, service_key_path: str):
-------
None
"""
- self.Initialize(service_account, service_key_path)
+ self.initialize(service_account, service_key_path)
pass
- def Initialize(self, service_account: str, service_key: str):
+ @staticmethod
+ def initialize(service_account: str, service_key: str):
"""Initialize.
Initialize authenticate and initializes the connection to google earth engine with a service accont file
@@ -47,10 +48,11 @@ def Initialize(self, service_account: str, service_key: str):
try:
credentials = ee.ServiceAccountCredentials(service_account, service_key)
except ValueError:
- credentials = ee.ServiceAccountCredentials(service_account, key_data=service_key)
+ credentials = ee.ServiceAccountCredentials(
+ service_account, key_data=service_key
+ )
ee.Initialize(credentials=credentials)
-
@staticmethod
def encodeServiceAccount(service_key_dir: str) -> bytes:
"""encodeServiceAccount.
@@ -70,7 +72,6 @@ def encodeServiceAccount(service_key_dir: str) -> bytes:
encoded_service_account = base64.b64encode(dumped_service_account.encode())
return encoded_service_account
-
@staticmethod
def decodeServiceAccount(service_key_bytes: bytes) -> str:
"""decodeServiceAccount.
diff --git a/earth2observe/gee/imagecollection.py b/earth2observe/gee/imagecollection.py
index a4a00d0..a386470 100644
--- a/earth2observe/gee/imagecollection.py
+++ b/earth2observe/gee/imagecollection.py
@@ -1,2 +1 @@
-
# collections = ee.ImageCollection(url')
diff --git a/earth2observe/gee/images.py b/earth2observe/gee/images.py
new file mode 100644
index 0000000..e69de29
diff --git a/earth2observe/utils.py b/earth2observe/utils.py
index 1ab00e5..9f57fec 100644
--- a/earth2observe/utils.py
+++ b/earth2observe/utils.py
@@ -1,15 +1,16 @@
-import sys
-import os
import gzip
+import os
+import sys
+
def print_progress_bar(
- i: int,
- total: int,
- prefix: str="",
- suffix: str="",
- decimals: int=1,
- length: int=100,
- fill: str="â"
+ i: int,
+ total: int,
+ prefix: str = "",
+ suffix: str = "",
+ decimals: int = 1,
+ length: int = 100,
+ fill: str = "â",
):
"""print_progress_bar.
@@ -45,11 +46,8 @@ def print_progress_bar(
print()
-
def extractFromGZ(input_file, output_file, delete=False):
- """
- ExtractFromGZ method extract data from the zip/.gz files,
- save the data
+ """ExtractFromGZ method extract data from the zip/.gz files, save the data.
Parameters
----------
@@ -73,4 +71,4 @@ def extractFromGZ(input_file, output_file, delete=False):
zf.close()
if delete:
- os.remove(input_file)
\ No newline at end of file
+ os.remove(input_file)
diff --git a/environment.yml b/environment.yml
index 17d354c..8cd6d58 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,16 +1,16 @@
channels:
- conda-forge
dependencies:
- - python >=3.7.1,<3.10
- - pip >=21.3.1
- - numpy >=1.21.2,<1.22.4
- - netCDF4 >=1.5.5,<1.5.9
+ - python >=3.9,<3.10
+ - pip >=22.2.2
+ - numpy >=1.21.2,<1.23.3
+ - netCDF4 >=1.5.5,<1.6.1
- gdal >=3.3.3,<3.5.1
- - pandas >=1.3.2,<1.4.3
- - pyramids >=0.1.0
- - ecmwf-api-client >=1.6.1
- - earthengine-api
+ - pandas >=1.3.2,<1.5.0
+ - pyramids >=0.2.2
+ - ecmwf-api-client >=1.6.3
+ - earthengine-api >=0.1.324
- pip:
- - loguru >=0.5.3
- - pytest >=6.2.5
- - pytest-cov >=2.12.1
+ - loguru >=0.6.0
+ - pytest >=7.1.3
+ - pytest-cov >=3.0.0
diff --git a/examples/Download Satellite data.ipynb b/examples/Download Satellite data.ipynb
index cc6c291..2448ba0 100644
--- a/examples/Download Satellite data.ipynb
+++ b/examples/Download Satellite data.ipynb
@@ -488,4 +488,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
-}
\ No newline at end of file
+}
diff --git a/examples/Download Satellite data.py b/examples/Download Satellite data.py
index ea55d01..a6f42ed 100644
--- a/examples/Download Satellite data.py
+++ b/examples/Download Satellite data.py
@@ -1,14 +1,12 @@
-"""
-Download Satellite data
-ECMWF
-Installation of ECMWF API key
+"""Download Satellite data ECMWF Installation of ECMWF API key.
+
1 - to be able to use Hapi to download ECMWF data you need to register and setup your account in the ECMWF website (https://apps.ecmwf.int/registration/)
2 - Install ECMWF key (instruction are here https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key)
"""
from earth2observe.chirps import CHIRPS
-from earth2observe.ecmwf import ECMWF
-from earth2observe.ecmwf import Variables
+from earth2observe.ecmwf import ECMWF, Variables
+
#%% precipitation
start = "2009-01-01"
end = "2009-01-10"
diff --git a/examples/data/ecmwf/data_interim.nc.aux.xml b/examples/data/ecmwf/data_interim.nc.aux.xml
new file mode 100644
index 0000000..f52faee
--- /dev/null
+++ b/examples/data/ecmwf/data_interim.nc.aux.xml
@@ -0,0 +1,56 @@
+
+
+
+
+ 2927.024999999818
+ 11951.97500000002
+ 60
+ 0
+ 0
+ 1|0|0|0|2|1|2|0|0|0|0|0|0|0|0|0|0|1|0|0|1|0|0|1|1|1|0|2|1|2|2|2|2|0|0|0|0|2|3|2|3|1|1|0|2|4|1|1|3|3|1|3|1|1|2|1|1|1|0|1
+
+
+
+ 11878
+ 8578.0333333333
+ 3001
+ 2229.0031999877
+
+
+
+
+
+ -15031.54999999978
+ -4594.449999999837
+ 60
+ 0
+ 0
+ 1|0|1|2|2|1|3|2|2|2|2|0|2|1|0|3|1|0|2|0|3|0|1|1|2|2|0|0|0|0|0|0|2|3|1|0|0|0|0|0|0|0|5|1|0|0|0|0|0|0|4|2|0|2|1|0|1|0|1|1
+
+
+
+ -4680
+ -10509.45
+ -14946
+ 3127.8967929745
+
+
+
+
+
+ -24705.23333333354
+ -4302.766666666692
+ 60
+ 0
+ 0
+ 1|0|1|1|0|2|1|1|3|1|1|3|1|2|1|1|2|0|1|2|1|2|0|0|2|2|0|2|0|1|1|2|0|1|1|0|1|2|2|0|2|1|0|1|2|1|1|0|1|1|0|1|0|2|0|0|0|1|0|1
+
+
+
+ -4470
+ -15696.383333333
+ -24538
+ 5399.3462693542
+
+
+
diff --git a/examples/data/linestring.geojson b/examples/data/linestring.geojson
new file mode 100644
index 0000000..3878729
--- /dev/null
+++ b/examples/data/linestring.geojson
@@ -0,0 +1,74 @@
+{
+ "type": "FeatureCollection",
+ "name": "linestring",
+ "crs": {
+ "type": "name",
+ "properties": {
+ "name": "urn:ogc:def:crs:OGC:1.3:CRS84"
+ }
+ },
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 1
+ },
+ "geometry": {
+ "type": "LineString",
+ "coordinates": [
+ [
+ -75.02773539668702,
+ 4.472264603312993
+ ],
+ [
+ -74.89270924149957,
+ 4.231963818657371
+ ],
+ [
+ -75.24972755013079,
+ 4.502016129032262
+ ],
+ [
+ -75.21997602441152,
+ 4.334949869224066
+ ],
+ [
+ -75.3000762859634,
+ 4.126689189189192
+ ],
+ [
+ -75.53808849171753,
+ 4.392164341761119
+ ]
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 2
+ },
+ "geometry": {
+ "type": "LineString",
+ "coordinates": [
+ [
+ -74.60663687881431,
+ 4.691968177855278
+ ],
+ [
+ -74.54484524847429,
+ 4.504304707933743
+ ],
+ [
+ -74.83320619006103,
+ 4.6507737576286
+ ],
+ [
+ -74.71648866608545,
+ 4.366989973844816
+ ]
+ ]
+ }
+ }
+ ]
+}
diff --git a/examples/data/point.gpkg b/examples/data/point.gpkg
new file mode 100644
index 0000000..b26f330
Binary files /dev/null and b/examples/data/point.gpkg differ
diff --git a/examples/data/points.geojson b/examples/data/points.geojson
new file mode 100644
index 0000000..da114e5
--- /dev/null
+++ b/examples/data/points.geojson
@@ -0,0 +1,90 @@
+{
+ "type": "FeatureCollection",
+ "name": "points",
+ "crs": {
+ "type": "name",
+ "properties": {
+ "name": "urn:ogc:def:crs:OGC:1.3:CRS84"
+ }
+ },
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 1
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -75.24972755013079,
+ 4.50201612903226
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 2
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -75.02544681778554,
+ 4.472264603312993
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 3
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -75.21997602441152,
+ 4.330372711421102
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 4
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -74.89499782040106,
+ 4.234252397558852
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 5
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -75.30694202266784,
+ 4.126689189189192
+ ]
+ }
+ },
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 6
+ },
+ "geometry": {
+ "type": "Point",
+ "coordinates": [
+ -75.53579991281606,
+ 4.392164341761119
+ ]
+ }
+ }
+ ]
+}
diff --git a/examples/data/points.qmd b/examples/data/points.qmd
new file mode 100644
index 0000000..872fba3
--- /dev/null
+++ b/examples/data/points.qmd
@@ -0,0 +1,26 @@
+
+
+
+
+
+ dataset
+
+
+
+
+
+
+
+
+
+ 0
+ 0
+
+
+
+
+ false
+
+
+
+
diff --git a/examples/data/polygon.geojson b/examples/data/polygon.geojson
new file mode 100644
index 0000000..89cee2d
--- /dev/null
+++ b/examples/data/polygon.geojson
@@ -0,0 +1,45 @@
+{
+ "type": "FeatureCollection",
+ "name": "polygon",
+ "crs": {
+ "type": "name",
+ "properties": {
+ "name": "urn:ogc:def:crs:OGC:1.3:CRS84"
+ }
+ },
+ "features": [
+ {
+ "type": "Feature",
+ "properties": {
+ "fid": 1
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [
+ -75.74692131647775,
+ 4.749182650392328
+ ],
+ [
+ -74.49850152571925,
+ 4.750326939843069
+ ],
+ [
+ -74.49964581517,
+ 4.000817349607672
+ ],
+ [
+ -75.75035418482997,
+ 4.000817349607672
+ ],
+ [
+ -75.74692131647775,
+ 4.749182650392328
+ ]
+ ]
+ ]
+ }
+ }
+ ]
+}
diff --git a/setup.py b/setup.py
index c5e4cb7..75cd8a0 100644
--- a/setup.py
+++ b/setup.py
@@ -3,27 +3,29 @@
with open("README.md", "r") as readme_file:
readme = readme_file.read()
-with open('HISTORY.rst') as history_file:
+with open("HISTORY.rst") as history_file:
history = history_file.read()
# requirements = [line.strip() for line in open("requirements.txt").readlines()]
# requirements = requirements[1:]
-test_requirements = ['pytest>=3', ]
+test_requirements = [
+ "pytest>=3",
+]
setup(
name="earth2observe",
- version="0.1.0",
+ version="0.1.1",
description="remote sensing package",
author="Mostafa Farrag",
author_email="moah.farag@gmail.come",
url="https://github.com/MAfarrag/earthobserve",
keywords=["remote sensing", "ecmwf"],
- long_description=readme + '\n\n' + history,
+ long_description=readme + "\n\n" + history,
long_description_content_type="text/markdown",
license="GNU General Public License v3",
zip_safe=False,
- packages=find_packages(include=['earth2observe', 'earth2observe.*']),
+ packages=find_packages(include=["earth2observe", "earth2observe.*"]),
test_suite="tests",
tests_require=test_requirements,
# install_requires=requirements,
@@ -35,9 +37,9 @@
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
- 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Natural Language :: English",
- 'Programming Language :: Python :: 3',
+ "Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
diff --git a/tests/conftest.py b/tests/conftest.py
index 1c03837..7369fbd 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -6,7 +6,6 @@
# from typing import List
-
# @pytest.fixture(scope="module")
# def time_series1() -> list:
- # return pd.read_csv("examples/data/time_series1.txt", header=None)[0].tolist()
+# return pd.read_csv("examples/data/time_series1.txt", header=None)[0].tolist()
diff --git a/tests/gee/conftest.py b/tests/gee/conftest.py
index dad867c..af8bff8 100644
--- a/tests/gee/conftest.py
+++ b/tests/gee/conftest.py
@@ -5,5 +5,17 @@
@pytest.fixture(scope="module")
def catalog_columns() -> List[str]:
- return ['dataset', 'name', 'provider', 'url', 'bands', 'band_describtion', 'spatial_resolution',
- 'temporal_resolution', 'start_date', 'end_date', 'min', 'max']
+ return [
+ "dataset",
+ "name",
+ "provider",
+ "url",
+ "bands",
+ "band_describtion",
+ "spatial_resolution",
+ "temporal_resolution",
+ "start_date",
+ "end_date",
+ "min",
+ "max",
+ ]