diff --git a/.gitignore b/.gitignore index 60465f6..48948a7 100644 --- a/.gitignore +++ b/.gitignore @@ -146,8 +146,8 @@ build_artifacts .idea mo_* conda/ -/examples/data/ecmwf/daily/* +/examples/data/ecmwf/* .idea/* examples/data/chirps/* *.xml -tests/mo/* \ No newline at end of file +tests/mo/* diff --git a/README.md b/README.md index 03e7a4f..8a8fb42 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Main Features ------------- - ERA Interim Download - CHIRPS Rainfall data Download + - ERA5 from Amason S3 data source Future work @@ -61,7 +62,7 @@ pip install git+https://github.com/MAfarrag/earthobserve ## pip to install the last release you can easly use pip ``` -pip install earthobserve==0.2.0 +pip install earthobserve==0.2.1 ``` Quick start diff --git a/conda-lock.yml b/conda-lock.yml index 8d01e6f..2cc0aea 100644 --- a/conda-lock.yml +++ b/conda-lock.yml @@ -15,9 +15,9 @@ metadata: - url: conda-forge used_env_vars: [] content_hash: - linux-64: 254a8ec847efd0a02a050ea9fc62f1a75932bb3645fa638fc79971d4b8f35430 - osx-64: d8e507a190f5b7c996b48cd0a82bc6c4ed5fed5a8279ee8d4157cb3c39be60c3 - win-64: cc58bac7c0731eff5cf8ea05e67e60e4ddf69f70dfdf9d0080a8a7faa1a4d316 + linux-64: 9eaae55afe9701810ae0e852f1b5fe4ebdffd940b4165beba9255cd4bf9ded7f + osx-64: e72b214899b7c355b81e32e383e1a018e3a880ac272dcc35efc0689971140455 + win-64: e6c1578e0dbf9fb2288f4763e9c778a9cfbd2c8d30dc770640394a5340b7e79e platforms: - linux-64 - osx-64 @@ -397,16 +397,16 @@ package: version: '20220623.0' - category: main dependencies: - libgcc-ng: '>=9.4.0' - libstdcxx-ng: '>=9.4.0' + libgcc-ng: '>=12' + libstdcxx-ng: '>=12' hash: - md5: c77f5e4e418fa47d699d6afa54c5d444 - sha256: f7c8866b27c4b6e2b2b84aae544fab539dfbfe5420c2c16fb868e9440bdb001e + md5: 0f683578378cddb223e7fd24f785ab2a + sha256: 4df6a29b71264fb25462065e8cddcf5bca60776b1801974af8cbd26b7425fcda manager: conda name: libaec optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-h9c3ff4c_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/linux-64/libaec-1.0.6-hcb278e6_1.conda version: 1.0.6 - category: main dependencies: @@ -437,14 +437,14 @@ package: dependencies: libgcc-ng: '>=12' hash: - md5: fc84a0446e4e4fb882e78d786cfb9734 - sha256: 6f7cbc9347964e7f9697bde98a8fb68e0ed926888b3116474b1224eaa92209dc + md5: 5cc781fd91968b11a8a7fdbee0982676 + sha256: f9983a8ea03531f2c14bce76c870ca325c0fddf0c4e872bff1f78bc52624179c manager: conda name: libdeflate optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.14-h166bdaf_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/linux-64/libdeflate-1.17-h0b41bf4_0.conda + version: '1.17' - category: main dependencies: libgcc-ng: '>=7.5.0' @@ -1214,7 +1214,7 @@ package: dependencies: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libgcc-ng: '>=12' libstdcxx-ng: '>=12' libwebp-base: '>=1.2.4,<2.0a0' @@ -1222,13 +1222,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: a01611c54334d783847879ee40109657 - sha256: 7237772229da1058fae73ae6f04ad846551a44d7da602e9d328b81049d3219a2 + md5: 2e648a34072eb39d7c4fc2a9981c5f0c + sha256: e3e18d91fb282b61288d4fd2574dfa31f7ae90ef2737f96722fb6ad3257862ee manager: conda name: libtiff optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h82bc61c_0.conda + url: https://conda.anaconda.org/conda-forge/linux-64/libtiff-4.5.0-h6adf6a1_2.conda version: 4.5.0 - category: main dependencies: @@ -1357,14 +1357,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -1515,6 +1515,18 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: libgcc-ng: '>=12' @@ -1769,14 +1781,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: libgcc-ng: '>=12' @@ -1806,18 +1818,30 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/linux-64/rtree-1.0.1-py310hbdcdc62_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -2877,7 +2901,7 @@ package: kealib: '>=1.5.0,<1.6.0a0' lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libgcc-ng: '>=12' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' @@ -2899,18 +2923,18 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' xerces-c: '>=3.2.4,<3.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: f16dc7517110a11910f7c1d9def32d38 - sha256: 84d5d701b0b739caa7a71eca47be2625079a2dd48c7638b9e5aedb72c916e593 + md5: 0a5408aac806b97ed97ca718957eb4b0 + sha256: c4a9694125c1ed214570df2e75ed1e799fa2aff8e85b365170ecd822d07aca22 manager: conda name: libgdal optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-he31f7c0_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/libgdal-3.6.2-h10cbb15_3.conda version: 3.6.2 - category: main dependencies: @@ -2949,20 +2973,20 @@ package: dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' libgcc-ng: '>=12' - libgdal: 3.6.2 he31f7c0_1 + libgdal: 3.6.2 h10cbb15_3 libstdcxx-ng: '>=12' numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' python_abi: 3.10.* *_cp310 hash: - md5: d6855fd7042a21cba318f68874e5b3e6 - sha256: fca1260c69715a21f12a0d5421495909e5bf09ab2b3662eb12b4ec6da5d871c3 + md5: 8f09d00becd16e21c92f26a99b231aac + sha256: eb3824b695871188e6b3a11f0fe4b950ba79327a6e33b7d69dd23fa85d7b49d6 manager: conda name: gdal optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py310hc1b7723_1.conda + url: https://conda.anaconda.org/conda-forge/linux-64/gdal-3.6.2-py310hc1b7723_3.conda version: 3.6.2 - category: main dependencies: @@ -3008,6 +3032,21 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: attrs: '>=17' @@ -3042,14 +3081,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: branca: '>=0.6.0' @@ -3101,6 +3140,34 @@ package: platform: linux-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: google-auth: '>=2.14.1,<3.0dev' @@ -3229,17 +3296,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: linux-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: @@ -3453,14 +3520,14 @@ package: - category: main dependencies: {} hash: - md5: ce2a6075114c9b64ad8cace52492feee - sha256: 0153de9987fa6e8dd5be45920470d579af433d4560bfd77318a72b3fd75fb6dc + md5: e3894420cf8b6abbf6c4d3d9742fbb4a + sha256: b322e190fd6fe631e1f4836ef99cbfb8352c03c30b51cb5baa216f7c9124d82e manager: conda name: libdeflate optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libdeflate-1.14-hb7f2c08_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/osx-64/libdeflate-1.17-hac1461d_0.conda + version: '1.17' - category: main dependencies: {} hash: @@ -3748,15 +3815,15 @@ package: version: '20220623.0' - category: main dependencies: - libcxx: '>=11.1.0' + libcxx: '>=14.0.6' hash: - md5: 0a49b696f11ed805ee4690479cc5e950 - sha256: 5c45ae356d10b6b78a9985e19d4cbd0e71cc76d1b43028f32737ef313ed525de + md5: 7c0f82f435ab4c48d65dc9b28db2ad9e + sha256: 38d32f4c7efddc204e53f43cd910122d3e6a997de1a3cd15f263217b225a9cdf manager: conda name: libaec optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libaec-1.0.6-he49afe7_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/osx-64/libaec-1.0.6-hf0c8a7f_1.conda version: 1.0.6 - category: main dependencies: @@ -4198,19 +4265,19 @@ package: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' libcxx: '>=14.0.6' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libwebp-base: '>=1.2.4,<2.0a0' libzlib: '>=1.2.13,<1.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: f32f9708c8d6f13e20a524c6da9a881e - sha256: 6659b6b71e79976e1bdd4b730a92425954e88cad6a184e274d6523d34d85f10e + md5: 35f714269a801f7c3cb522aacd3c0e69 + sha256: 03d00d6a3b1e569e9a8da66a9ad75a29c9c676dc7de6c16771abbb961abded2c manager: conda name: libtiff optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libtiff-4.5.0-h6268bbc_0.conda + url: https://conda.anaconda.org/conda-forge/osx-64/libtiff-4.5.0-hee9004a_2.conda version: 4.5.0 - category: main dependencies: @@ -4331,14 +4398,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -4486,6 +4553,18 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: libcxx: '>=14.0.4' @@ -4743,14 +4822,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -4779,18 +4858,30 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/osx-64/rtree-1.0.1-py310had9ce37_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -5705,7 +5796,7 @@ package: lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' libcxx: '>=14.0.6' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' libnetcdf: '>=4.8.1,<4.8.2.0a0' @@ -5724,18 +5815,18 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' xerces-c: '>=3.2.4,<3.3.0a0' xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: 1eb9593ce19fa949f7c32cb289271e82 - sha256: 61d2094cc01664cedcf23ab34616802b63ed31d65c520815977c8999f4f27ee6 + md5: 1ae43833d5a0c8d7504a991eb13e2439 + sha256: f077161cbbfe41d687804237adb169949ed911432e4a23786aab1fa017ef1b4a manager: conda name: libgdal optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libgdal-3.6.2-h44a409f_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/libgdal-3.6.2-h623d8b8_3.conda version: 3.6.2 - category: main dependencies: @@ -5800,19 +5891,19 @@ package: dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' libcxx: '>=14.0.6' - libgdal: 3.6.2 h44a409f_1 + libgdal: 3.6.2 h623d8b8_3 numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' python_abi: 3.10.* *_cp310 hash: - md5: 16d96cc0855982325700c8623af85ffb - sha256: ddebc8a3f9ad251ed8acbfbf0db0cf4af93fdc1264ad5757d64baa586a783e86 + md5: eebf4977fcaa1f20f94b55af26110eab + sha256: 043c842b3d96f3aa4ce491f12f8b5510aade83d74566655894f18d81837cfc81 manager: conda name: gdal optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/gdal-3.6.2-py310h5abc6fc_1.conda + url: https://conda.anaconda.org/conda-forge/osx-64/gdal-3.6.2-py310h5abc6fc_3.conda version: 3.6.2 - category: main dependencies: @@ -5878,6 +5969,21 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: attrs: '>=17' @@ -5997,14 +6103,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: branca: '>=0.6.0' @@ -6056,6 +6162,34 @@ package: platform: osx-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: osx-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: google-auth: '>=2.14.1,<3.0dev' @@ -6183,17 +6317,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: osx-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: @@ -6580,16 +6714,17 @@ package: version: '20220623.0' - category: main dependencies: - vc: '>=14.1,<15.0a0' - vs2015_runtime: '>=14.16.27012' + ucrt: '>=10.0.20348.0' + vc: '>=14.2,<15' + vs2015_runtime: '>=14.29.30139' hash: - md5: ac78b243f1ee03a2412b6e328aa3a12d - sha256: 47f2ef0486d690b2bc34035a165a86c1edcc18d48f1f8aa8eead594afa0dfebf + md5: f98474a8245f55f4a273889dbe7bf193 + sha256: 441f580f90279bd62bd27fb82d0bbbb2c2d9f850fcc4c8781f199c5287cd1499 manager: conda name: libaec optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libaec-1.0.6-h39d44d4_0.tar.bz2 + url: https://conda.anaconda.org/conda-forge/win-64/libaec-1.0.6-h63175ca_1.conda version: 1.0.6 - category: main dependencies: @@ -6620,17 +6755,18 @@ package: version: 1.1.2 - category: main dependencies: + ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' hash: - md5: 4366e00d3270eb229c026920474a6dda - sha256: c8b156fc81006234cf898f933b06bed8bb475970cb7983d0eceaf90db65beb8b + md5: ae9dfb57bcb42093a2417aceabb530f7 + sha256: 76e642ca8a11da1b537506447f8089353b6607956c069c938a4bec4de36e1194 manager: conda name: libdeflate optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libdeflate-1.14-hcfcfb64_0.tar.bz2 - version: '1.14' + url: https://conda.anaconda.org/conda-forge/win-64/libdeflate-1.17-hcfcfb64_0.conda + version: '1.17' - category: main dependencies: vc: '>=14.1,<15.0a0' @@ -7221,14 +7357,14 @@ package: dependencies: python: '>=3.7' hash: - md5: c6653a1ed0c4a48ace64ab68a0bf9b27 - sha256: ae9d26949fcf8130d899e6bc22ed8afab40adcee782d79e0d82e0799960785af + md5: b181b206ebe85661209ccb542d21845f + sha256: 27f2a67e3bcc185791d870c1f4299163ac35f2842a5f2ca607539e9e6f9668a6 manager: conda name: cachetools optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.0-pyhd8ed1ab_0.tar.bz2 - version: 5.2.0 + url: https://conda.anaconda.org/conda-forge/noarch/cachetools-5.2.1-pyhd8ed1ab_0.conda + version: 5.2.1 - category: main dependencies: python: '>=3.7' @@ -7397,6 +7533,18 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda version: 2.0.0 +- category: main + dependencies: + python: '>=3.7' + hash: + md5: 2cfa3e1cf3fb51bb9b17acc5b5e9ea11 + sha256: 95ac5f9ee95fd4e34dc051746fc86016d3d4f6abefed113e2ede049d59ec2991 + manager: conda + name: jmespath + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/jmespath-1.0.1-pyhd8ed1ab_0.tar.bz2 + version: 1.0.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -7506,7 +7654,7 @@ package: dependencies: jpeg: '>=9e,<10a' lerc: '>=4.0.0,<5.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libzlib: '>=1.2.13,<1.3.0a0' ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' @@ -7514,13 +7662,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: d9b568beb74c97e53dbb9531b14f69c0 - sha256: dc13f42b392e05a1e705ceebcde82ed3e15663b511e78585d26fc4b9bf628b59 + md5: 2e003e276cc1375192569c96afd3d984 + sha256: 86cf8066db11f84b506ba246944901584ab199dfe7490586f5e9b6c299e3b8e0 manager: conda name: libtiff optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libtiff-4.5.0-hc4f729c_0.conda + url: https://conda.anaconda.org/conda-forge/win-64/libtiff-4.5.0-hf8721a0_2.conda version: 4.5.0 - category: main dependencies: @@ -7667,14 +7815,14 @@ package: dependencies: python: '>=3.6' hash: - md5: c8d7e34ca76d6ecc03b84bedfd99d689 - sha256: 000f38e7ce7f020e2ce4d5024d3ffa63fcd65077edfe2182862965835f560525 + md5: f59d49a7b464901cf714b9e7984d01a2 + sha256: 93cfc7a92099e26b0575a343da4a667b52371cc38e4dee4ee264dc041ef77bac manager: conda name: pytz optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7-pyhd8ed1ab_0.conda - version: '2022.7' + url: https://conda.anaconda.org/conda-forge/noarch/pytz-2022.7.1-pyhd8ed1ab_0.conda + version: 2022.7.1 - category: main dependencies: python: '>=3.10,<3.11.0a0' @@ -7706,18 +7854,30 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/win-64/rtree-1.0.1-py310h1cbd46b_1.tar.bz2 version: 1.0.1 +- category: main + dependencies: + python: '>=3.9,<3.11' + hash: + md5: 4b4fdbb5d12180a49f5ffc1f5a572c2c + sha256: 5bc0500f3ac7e60a3877c81083338d0ac301376263919a78dd4bdd007166a377 + manager: conda + name: serapeum_utils + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/serapeum_utils-0.1.0-pyhd8ed1ab_0.conda + version: 0.1.0 - category: main dependencies: python: '>=3.7' hash: - md5: 9600fc9524d3f821e6a6d58c52f5bf5a - sha256: ea9f7eee2648d8078391cf9f968d848b400349c784e761501fb32ae01d323acf + md5: 88ec352fde1a5561ce5f93a302539f7f + sha256: c0550e243de0503f4617d564259f44561619b754407dd27f49e420565148e82b manager: conda name: setuptools optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-65.6.3-pyhd8ed1ab_0.conda - version: 65.6.3 + url: https://conda.anaconda.org/conda-forge/noarch/setuptools-66.0.0-pyhd8ed1ab_0.conda + version: 66.0.0 - category: main dependencies: python: '' @@ -8845,7 +9005,7 @@ package: kealib: '>=1.5.0,<1.6.0a0' lerc: '>=4.0.0,<5.0a0' libcurl: '>=7.87.0,<8.0a0' - libdeflate: '>=1.14,<1.15.0a0' + libdeflate: '>=1.17,<1.18.0a0' libiconv: '>=1.17,<2.0a0' libkml: '>=1.3.0,<1.4.0a0' libnetcdf: '>=4.8.1,<4.8.2.0a0' @@ -8864,7 +9024,7 @@ package: poppler: '>=22.12.0,<22.13.0a0' postgresql: '' proj: '>=9.1.0,<9.1.1.0a0' - tiledb: '>=2.13.1,<2.14.0a0' + tiledb: '>=2.13.2,<2.14.0a0' ucrt: '>=10.0.20348.0' vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' @@ -8872,13 +9032,13 @@ package: xz: '>=5.2.6,<6.0a0' zstd: '>=1.5.2,<1.6.0a0' hash: - md5: 9a4ef5cd8e935b794df7aea0cc4ca70a - sha256: 140e1141816bd5e7a560fef469ca435394d07eea5f9f37d01c18be59cee1a603 + md5: a73eceda4d87ffc2ae1e6d151f3f878a + sha256: 6dcd51c0c30b3c788b207ebc1ce716b27218333a2aa81e130d2e85bf26681d42 manager: conda name: libgdal optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/libgdal-3.6.2-hffd0036_1.conda + url: https://conda.anaconda.org/conda-forge/win-64/libgdal-3.6.2-h060c9ed_3.conda version: 3.6.2 - category: main dependencies: @@ -8910,6 +9070,21 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.14-pyhd8ed1ab_0.conda version: 1.26.14 +- category: main + dependencies: + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + python-dateutil: '>=2.1,<3.0.0' + urllib3: '>=1.25.4,<1.27' + hash: + md5: 948b7598f77ef952b30516047b2614fe + sha256: e0b36c72243fe5af0d65b5a7330ef30d169e6190fd1996650e3f95314fdafec0 + manager: conda + name: botocore + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/botocore-1.29.50-pyhd8ed1ab_0.conda + version: 1.29.50 - category: main dependencies: libblas: '>=3.9.0,<4.0a0' @@ -8937,14 +9112,14 @@ package: python: '>=3.7,<4.0' urllib3: '>=1.21.1,<1.27' hash: - md5: 089382ee0e2dc2eae33a04cc3c2bddb0 - sha256: b45d0da6774c8231ab4fef0427b3050e7c54c84dfe453143dd4010999c89e050 + md5: 11d178fc55199482ee48d6812ea83983 + sha256: 22c081b4cdd023a514400413f50efdf2c378f56f2a5ea9d65666aacf4696490a manager: conda name: requests optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.1-pyhd8ed1ab_1.tar.bz2 - version: 2.28.1 + url: https://conda.anaconda.org/conda-forge/noarch/requests-2.28.2-pyhd8ed1ab_0.conda + version: 2.28.2 - category: main dependencies: numpy: '>=1.21.6,<2.0a0' @@ -8998,7 +9173,7 @@ package: - category: main dependencies: hdf5: '>=1.12.2,<1.12.3.0a0' - libgdal: 3.6.2 hffd0036_1 + libgdal: 3.6.2 h060c9ed_3 numpy: '>=1.21.6,<2.0a0' openssl: '>=3.0.7,<4.0a0' python: '>=3.10,<3.11.0a0' @@ -9007,13 +9182,13 @@ package: vc: '>=14.2,<15' vs2015_runtime: '>=14.29.30139' hash: - md5: ed437558c068417a3fedd81f51aaa828 - sha256: 6c778b866106fe1cbe7560982705ad4f280fe42b1fb9dc290cd4b978c535a55c + md5: 8934913fb065c580d1d78b55fd1e9b42 + sha256: a40a315a226c12cc1eace77d58220f9120f37027d17fa2f8698fd937fe95a725 manager: conda name: gdal optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/win-64/gdal-3.6.2-py310h644bc08_1.conda + url: https://conda.anaconda.org/conda-forge/win-64/gdal-3.6.2-py310h644bc08_3.conda version: 3.6.2 - category: main dependencies: @@ -9069,6 +9244,19 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/pooch-1.6.0-pyhd8ed1ab_0.tar.bz2 version: 1.6.0 +- category: main + dependencies: + botocore: '>=1.12.36,<2.0a.0' + python: '>=3.7' + hash: + md5: 900e74d8547fbea3af028937df28ed77 + sha256: 0e459ed32b00e96b62c2ab7e2dba0135c73fd980120fe1a7bd49901f2d50760f + manager: conda + name: s3transfer + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/s3transfer-0.6.0-pyhd8ed1ab_0.tar.bz2 + version: 0.6.0 - category: main dependencies: geos: '>=3.11.1,<3.11.2.0a0' @@ -9101,6 +9289,21 @@ package: platform: win-64 url: https://conda.anaconda.org/conda-forge/noarch/snuggs-1.4.7-py_0.tar.bz2 version: 1.4.7 +- category: main + dependencies: + botocore: '>=1.29.50,<1.30.0' + jmespath: '>=0.7.1,<2.0.0' + python: '>=3.7' + s3transfer: '>=0.6.0,<0.7.0' + hash: + md5: ca3fd345a15e696347a25301a5a13eaa + sha256: 6f9acded0e0468416b467c428b3d6e0920d5558078e14ecf4a07ff05e41aa2db + manager: conda + name: boto3 + optional: false + platform: win-64 + url: https://conda.anaconda.org/conda-forge/noarch/boto3-1.26.50-pyhd8ed1ab_0.conda + version: 1.26.50 - category: main dependencies: attrs: '>=17' @@ -9349,17 +9552,17 @@ package: networkx: '' numpy: '>=1.3' pandas: '>=1.0' - python: '>=3.5' + python: '>=3.6' scikit-learn: '' scipy: '>=1.0' hash: - md5: 2b0aace000c60c7a6de29907e3df4c6e - sha256: 76db2b87e74d94ff0ab64e9d5809d853f5f2735ee205e135b982a93bed72978d + md5: db1aeaff6e248db425e049feffded7a9 + sha256: 78aadbd9953976678b6e3298ac26a63cf9390a8794db3ff71f3fe5b6d13a35ca manager: conda name: mapclassify optional: false platform: win-64 - url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/mapclassify-2.5.0-pyhd8ed1ab_1.conda version: 2.5.0 - category: main dependencies: diff --git a/earth2observe/abstractdatasource.py b/earth2observe/abstractdatasource.py index f28ebea..c635750 100644 --- a/earth2observe/abstractdatasource.py +++ b/earth2observe/abstractdatasource.py @@ -1,4 +1,6 @@ +import os from abc import ABC, abstractmethod +from pathlib import Path from typing import Dict @@ -14,7 +16,7 @@ def __init__( lat_lim: list = None, lon_lim: list = None, fmt: str = "%Y-%m-%d", - # path: str = "", + path: str = "", ): """ @@ -45,20 +47,27 @@ def __init__( self.create_grid(lat_lim, lon_lim) self.check_input_dates(start, end, temporal_resolution, fmt) + self.path = Path(path).absolute() + # Create the directory + if not os.path.exists(self.path): + os.makedirs(self.path) + pass @abstractmethod - def check_input_dates(self): + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): """Check validity of input dates.""" pass @abstractmethod - def initialize(self): + def initialize(self, *args, **kwargs): """Initialize connection with the data source server (for non ftp servers)""" pass @abstractmethod - def create_grid(self): + def create_grid(self, lat_lim: list, lon_lim: list): """create a grid from the lat/lon boundaries.""" pass @@ -84,6 +93,9 @@ def API(self): class AbstractCatalog(ABC): """abstrach class for the datasource catalog.""" + def __init__(self): + self.catalog = self.get_catalog() + @abstractmethod def get_catalog(self): """read the catalog of the datasource from disk or retrieve it from server.""" @@ -92,4 +104,4 @@ def get_catalog(self): @abstractmethod def get_variable(self, var_name) -> Dict[str, str]: """get the details of a specific variable.""" - pass + return self.catalog.get(var_name) diff --git a/earth2observe/chirps.py b/earth2observe/chirps.py index 29b554e..d7b0dc6 100644 --- a/earth2observe/chirps.py +++ b/earth2observe/chirps.py @@ -1,7 +1,6 @@ import datetime as dt import os from ftplib import FTP -from pathlib import Path import numpy as np import pandas as pd @@ -9,9 +8,9 @@ from osgeo import gdal from pyramids.raster import Raster from pyramids.utils import extractFromGZ +from serapeum_utils.utils import print_progress_bar from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from earth2observe.utils import print_progress_bar class CHIRPS(AbstractDataSource): @@ -66,14 +65,8 @@ def __init__( lat_lim=lat_lim, lon_lim=lon_lim, fmt=fmt, + path=path, ) - self.output_folder = os.path.join( - Path(path).absolute(), "chirps", "precipitation" - ) - - # make directory if it not exists - if not os.path.exists(self.output_folder): - os.makedirs(self.output_folder) def check_input_dates( self, start: str, end: str, temporal_resolution: str, fmt: str @@ -105,10 +98,10 @@ def check_input_dates( # Define timestep for the timedates if temporal_resolution.lower() == "daily": self.time_freq = "D" - # self.output_folder = os.path.join(path, "precipitation", "chirps", "daily") + # self.path = os.path.join(path, "precipitation", "chirps", "daily") elif temporal_resolution.lower() == "monthly": self.time_freq = "MS" - # self.output_folder = os.path.join( + # self.path = os.path.join( # path, "Precipitation", "CHIRPS", "Monthly" # ) else: @@ -189,7 +182,7 @@ def download(self, progress_bar: bool = True, cores=None, *args, **kwargs): """ # Pass variables to parallel function and run args = [ - self.output_folder, + self.path, self.temporal_resolution, self.xID, self.yID, @@ -238,7 +231,7 @@ def API(self, date, args): args: [list] """ - [output_folder, temp_resolution, xID, yID, lon_lim, latlim] = args + [path, temp_resolution, xID, yID, lon_lim, latlim] = args # Define FTP path to directory if temp_resolution.lower() == "daily": @@ -252,11 +245,11 @@ def API(self, date, args): if temp_resolution.lower() == "daily": filename = f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif.gz" outfilename = os.path.join( - output_folder, + path, f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) DirFileEnd = os.path.join( - output_folder, + path, f"{self.clipped_fname}_mm-day-1_daily_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) elif temp_resolution == "monthly": @@ -264,23 +257,23 @@ def API(self, date, args): f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif.gz" ) outfilename = os.path.join( - output_folder, + path, f"{self.globe_fname}.{date.strftime('%Y')}.{date.strftime('%m')}.tif", ) DirFileEnd = os.path.join( - output_folder, + path, f"{self.clipped_fname}_mm-month-1_monthly_{date.strftime('%Y')}.{date.strftime('%m')}.{date.strftime('%d')}.tif", ) else: raise KeyError("The input temporal_resolution interval is not supported") - self.callAPI(pathFTP, output_folder, filename) + self.callAPI(pathFTP, path, filename) self.post_download( - output_folder, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd + path, filename, lon_lim, latlim, xID, yID, outfilename, DirFileEnd ) @staticmethod - def callAPI(pathFTP: str, output_folder: str, filename: str): + def callAPI(pathFTP: str, path: str, filename: str): """send the request to the server. RetrieveData method retrieves CHIRPS data for a given date from the @@ -289,7 +282,7 @@ def callAPI(pathFTP: str, output_folder: str, filename: str): Parameters ---------- filename - output_folder + path pathFTP @@ -313,14 +306,14 @@ def callAPI(pathFTP: str, output_folder: str, filename: str): ftp.retrlines("LIST", listing.append) # download the global rainfall file - local_filename = os.path.join(output_folder, filename) + local_filename = os.path.join(path, filename) lf = open(local_filename, "wb") ftp.retrbinary("RETR " + filename, lf.write, 8192) lf.close() def post_download( self, - output_folder, + path, filename, lon_lim, latlim, @@ -333,7 +326,7 @@ def post_download( Parameters ---------- - output_folder: [str] + path: [str] directory where files will be saved filename: [str] file name @@ -346,7 +339,7 @@ def post_download( """ try: # unzip the file - zip_filename = os.path.join(output_folder, filename) + zip_filename = os.path.join(path, filename) extractFromGZ(zip_filename, outfilename, delete=True) # open tiff file @@ -397,14 +390,14 @@ class Catalog(AbstractCatalog): """CHIRPS data catalog.""" def __init__(self): - self.catalog = self.get_catalog() + super().__init__() def get_catalog(self): """return the catalog.""" return { "Precipitation": { - "descriptions": "rainfall [mm/temporal_resolution step]", - "units": "mm/temporal_resolution step", + "descriptions": "rainfall [mm/temporal_resolution]", + "units": "mm/temporal_resolution", "temporal resolution": ["daily", "monthly"], "file name": "rainfall", "var_name": "R", @@ -413,4 +406,4 @@ def get_catalog(self): def get_variable(self, var_name): """get the details of a specific variable.""" - return self.catalog.get(var_name) + return super().get_variable(var_name) diff --git a/earth2observe/earth2observe.py b/earth2observe/earth2observe.py index 6ed6ab4..421cbd5 100644 --- a/earth2observe/earth2observe.py +++ b/earth2observe/earth2observe.py @@ -1,12 +1,16 @@ """Front end module that runs each data source backend.""" from earth2observe.chirps import CHIRPS from earth2observe.ecmwf import ECMWF +from earth2observe.s3 import S3 + +DEFAULT_LONGITUDE_LIMIT = [-180, 180] +DEFAULT_LATITUDE_LIMIT = [-90, 90] class Earth2Observe: """End user class to call all the data source classes abailable in earth2observe.""" - DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS} + DataSources = {"ecmwf": ECMWF, "chirps": CHIRPS, "amazon-s3": S3} def __init__( self, @@ -22,6 +26,10 @@ def __init__( ): if data_source not in self.DataSources: raise ValueError(f"{data_source} not supported") + if lat_lim is None: + lat_lim = DEFAULT_LATITUDE_LIMIT + if lon_lim is None: + lon_lim = DEFAULT_LONGITUDE_LIMIT self.datasource = self.DataSources[data_source]( start=start, diff --git a/earth2observe/ecmwf.py b/earth2observe/ecmwf.py index 8c2dce4..2850906 100644 --- a/earth2observe/ecmwf.py +++ b/earth2observe/ecmwf.py @@ -5,7 +5,6 @@ import calendar import datetime as dt import os -from pathlib import Path from typing import Dict import numpy as np @@ -15,10 +14,10 @@ from loguru import logger from netCDF4 import Dataset from pyramids.raster import Raster +from serapeum_utils.utils import print_progress_bar from earth2observe import __path__ from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource -from earth2observe.utils import print_progress_bar class ECMWF(AbstractDataSource): @@ -70,8 +69,8 @@ def __init__( lat_lim=lat_lim, lon_lim=lon_lim, fmt=fmt, + path=path, ) - self.path = Path(path).absolute() def check_input_dates( self, start: str, end: str, temporal_resolution: str, fmt: str @@ -169,9 +168,7 @@ def download( f"Download ECMWF {var} data for period {self.start} till {self.end}" ) var_info = catalog.get_variable(var) - self.downloadDataset( - var_info, dataset=dataset, progress_bar=progress_bar - ) # CaseParameters=[SumMean, Min, Max] + self.downloadDataset(var_info, dataset=dataset, progress_bar=progress_bar) # delete the downloaded netcdf del_ecmwf_dataset = os.path.join(self.path, "data_interim.nc") os.remove(del_ecmwf_dataset) @@ -207,16 +204,10 @@ def downloadDataset( progress_bar: [bool] True if you want to display a progress bar. """ - # Create the directory - out_dir = f"{self.path}/{self.temporal_resolution}/{var_info.get('file name')}" - - if not os.path.exists(out_dir): - os.makedirs(out_dir) - # trigger the request to the server self.API(var_info, dataset) # process the downloaded data - self.post_download(var_info, out_dir, dataset, progress_bar) + self.post_download(var_info, self.path, dataset, progress_bar) def API(self, var_info, dataset): """form the request url abd trigger the request. @@ -331,9 +322,6 @@ def callAPI( area_str dataset """ - - os.chdir(output_folder) - if download_type == 1 or download_type == 2: server.retrieve( { @@ -349,7 +337,7 @@ def callAPI( "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": area_str, "format": "netcdf", - "target": f"data_{dataset}.nc", + "target": f"{output_folder}/data_{dataset}.nc", } ) @@ -369,7 +357,7 @@ def callAPI( "class": class_str, # http://apps.ecmwf.int/codes/grib/format/mars/class/ "area": area_str, "format": "netcdf", - "target": f"data_{dataset}.nc", + "target": f"{output_folder}/data_{dataset}.nc", } ) @@ -482,7 +470,7 @@ def post_download( # Define the out name name_out = os.path.join( out_dir, - f"%{var_output_name}_ECMWF_ERA-Interim_{Var_unit}_{self.temporal_resolution}_{year}.{month}.{day}.tif", + f"{var_output_name}_ECMWF_ERA-Interim_{Var_unit}_{self.temporal_resolution}_{year}.{month}.{day}.tif", ) # Create Tiff files @@ -505,10 +493,10 @@ def post_download( class Catalog(AbstractCatalog): """ECMWF data catalog This class contains the information about the ECMWF variables http://rda.ucar.edu/cgi-bin/transform?xml=/metadata/ParameterTables/WMO_GRIB1.98-0.128.xml&view=gribdoc.""" - def __init__(self, version: int = 1): + def __init__(self): # get the catalog - self.catalog = self.get_catalog() - self.version = version + super().__init__() + pass def get_catalog(self): """readthe data catalog from disk.""" @@ -518,7 +506,7 @@ def get_catalog(self): def get_variable(self, var_name): """retrieve a variable form the datasource catalog.""" - return self.catalog.get(var_name) + return super().get_variable(var_name) class AuthenticationError(Exception): diff --git a/earth2observe/gee/dataset.py b/earth2observe/gee/dataset.py index 0d021fa..67f7847 100644 --- a/earth2observe/gee/dataset.py +++ b/earth2observe/gee/dataset.py @@ -59,10 +59,10 @@ def getDate( Returns ------- start_date: [str] - beginning of the time series. + beginning of the temporal_resolution series. end_date: [str] - end of the time series. + end of the temporal_resolution series. """ data = catalog.loc[catalog["dataset"] == dataset_id, :] diff --git a/earth2observe/s3.py b/earth2observe/s3.py new file mode 100644 index 0000000..3c22926 --- /dev/null +++ b/earth2observe/s3.py @@ -0,0 +1,396 @@ +"""Amazon S3.""" +import datetime as dt +import os +from typing import Dict, List + +import boto3 +import botocore +import pandas as pd +from botocore import exceptions +from serapeum_utils.utils import print_progress_bar + +from earth2observe.abstractdatasource import AbstractCatalog, AbstractDataSource + + +class S3(AbstractDataSource): + """Amazon S3 data source.""" + + def __init__( + self, + temporal_resolution: str = "monthly", + start: str = None, + end: str = None, + path: str = "", + variables: list = "precipitation_amount_1hour_Accumulation", + lat_lim: list = None, + lon_lim: list = None, + fmt: str = "%Y-%m-%d", + ): + """S3. + + Parameters + ---------- + temporal_resolution (str, optional): + 'daily' or 'monthly'. Defaults to 'daily'. + start (str, optional): + [description]. Defaults to ''. + end (str, optional): + [description]. Defaults to ''. + path (str, optional): + Path where you want to save the downloaded data. Defaults to ''. + variables (list, optional): + Variable code: VariablesInfo('day').descriptions.keys(). Defaults to []. + lat_lim (list, optional): + [ymin, ymax] (values must be between -50 and 50). Defaults to []. + lon_lim (list, optional): + [xmin, xmax] (values must be between -180 and 180). Defaults to []. + fmt (str, optional): + [description]. Defaults to "%Y-%m-%d". + """ + super().__init__( + start=start, + end=end, + variables=variables, + temporal_resolution=temporal_resolution, + lat_lim=lat_lim, + lon_lim=lon_lim, + fmt=fmt, + path=path, + ) + + def initialize(self, bucket: str = "era5-pds") -> object: + """initialize connection with amazon s3 and create a client. + + Parameters + ---------- + bucket: [str] + S3 bucket name. + + Returns + ------- + client: [botocore.client.S3] + Amazon S3 client + """ + # AWS access / secret keys required + # s3 = boto3.resource('s3') + # bucket = s3.Bucket(era5_bucket) + + # No AWS keys required + client = boto3.client( + "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) + ) + self.client = client + return client + + def create_grid(self, lat_lim: list, lon_lim: list): + """TODO:""" + pass + + def check_input_dates( + self, start: str, end: str, temporal_resolution: str, fmt: str + ): + """check validity of input dates. + + Parameters + ---------- + temporal_resolution: (str, optional) + [description]. Defaults to 'daily'. + start: (str, optional) + [description]. Defaults to ''. + end: (str, optional) + [description]. Defaults to ''. + fmt: (str, optional) + [description]. Defaults to "%Y-%m-%d". + """ + self.start = dt.datetime.strptime(start, fmt) + self.end = dt.datetime.strptime(end, fmt) + + # Set required data for the daily option + if temporal_resolution == "daily": + self.dates = pd.date_range(self.start, self.end, freq="D") + elif temporal_resolution == "monthly": + self.dates = pd.date_range(self.start, self.end, freq="MS") + + def download(self, progress_bar: bool = True): + """Download wrapper over all given variables. + + ECMWF method downloads ECMWF daily data for a given variable, temporal_resolution + interval, and spatial extent. + + + Parameters + ---------- + progress_bar : TYPE, optional + 0 or 1. to display the progress bar + dataset:[str] + Default is "interim" + + Returns + ------- + None. + """ + catalog = Catalog() + for var in self.vars: + var_info = catalog.get_variable(var) + var_s3_name = var_info.get("bucket_name") + self.downloadDataset(var_s3_name, progress_bar=progress_bar) + + def downloadDataset(self, var: str, progress_bar: bool = True): + """Download a climate variable. + + This function downloads ECMWF six-hourly, daily or monthly data. + + Parameters + ---------- + var: [str] + variable detailed information + >>> { + >>> 'descriptions': 'Evaporation [m of water]', + >>> 'units': 'mm', + >>> 'types': 'flux', + >>> 'temporal resolution': ['six hours', 'daily', 'monthly'], + >>> 'file name': 'Evaporation', + >>> } + progress_bar: [bool] + True if you want to display a progress bar. + """ + if progress_bar: + total_amount = len(self.dates) + amount = 0 + print_progress_bar( + amount, + total_amount, + prefix="Progress:", + suffix="Complete", + length=50, + ) + + for date in self.dates: + year = date.strftime("%Y") + month = date.strftime("%m") + # file path patterns for remote S3 objects and corresponding local file + s3_data_key = f"{year}/{month}/data/{var}.nc" + downloaded_file_dir = ( + f"{self.path}/{year}{month}_{self.temporal_resolution}_{var}.nc" + ) + + self.API(s3_data_key, downloaded_file_dir) + + if progress_bar: + amount = amount + 1 + print_progress_bar( + amount, + total_amount, + prefix="Progress:", + suffix="Complete", + length=50, + ) + + def API(self, s3_file_path: str, local_dir_fname: str, bucket: str = "era5-pds"): + """Download file from s3 bucket. + + Parameters + ---------- + s3_file_path: str + the whole path for the file inside the bcket. i.e. "2022/02/main.nc" + local_dir_fname: [str] + absolute path for the file name and directory in your local drive. + bucket: [str] + bucket name. Default is "era5-pds" + + Returns + ------- + Download the file to your local drive. + """ + if not os.path.isfile(local_dir_fname): # check if file already exists + print(f"Downloading {s3_file_path} from S3...") + try: + self.client.download_file(bucket, s3_file_path, local_dir_fname) + except exceptions.ClientError: + print( + f"Error while downloading the {s3_file_path} please check the file name" + ) + else: + print(f"The file {local_dir_fname} already in your local directory") + + @staticmethod + def parse_response_metadata(response: Dict[str, str]): + """parse client response. + + Parameters + ---------- + response: + >>> { + >>> 'RequestId': 'E01V3VYXERE4JV8Z', + >>> 'HostId': 'Nb33s2L/qsQ/oMb8tyT737ymO5sNnAv+KeKEbhvjILbDPvUi0sDVj6zbuRPh/kpmK5BY6Y3EK/A=', + >>> 'HTTPStatusCode': 200, + >>> 'HTTPHeaders': {'x-amz-id-2': 'Nb33s2L/qsQ/oMb8tyT737ymO5sNnAv+KeKEbhvjILbDPvUi0sDVj6zbuRPh/kpmK5BY6Y3EK/A=', + >>> 'x-amz-request-id': 'E01V3VYXERE4JV8Z', + >>> 'date': 'Sun, 15 Jan 2023 22:36:28 GMT', + >>> 'x-amz-bucket-region': 'us-east-1', + >>> 'content-type': 'application/xml', + >>> 'transfer-encoding': 'chunked', + >>> 'server': 'AmazonS3'}, + >>> 'RetryAttempts': 0 + >>> } + """ + response_meta = response.get("ResponseMetadata") + keys = [] + if response_meta.get("HTTPStatusCode") == 200: + contents_list = response.get("Contents") + if contents_list is None: + print("No objects are available") # {date.strftime('%B, %Y')} + else: + for obj in contents_list: + keys.append(obj.get("Key")) + print( + f"There are {len(keys)} objects available for\n--" + ) # {date.strftime('%B, %Y')} + for k in keys: + print(k) + else: + print("There was an error with your request.") + + return keys + + +class Catalog(AbstractCatalog): + """S3 data catalog.""" + + def __init__(self, bucket: str = "era5-pds"): + """ + + Parameters + ---------- + bucket : + """ + super().__init__() + self.client = self.initialize(bucket=bucket) + + @staticmethod + def initialize(bucket: str = "era5-pds") -> object: + """initialize connection with amazon s3 and create a client. + + Parameters + ---------- + bucket: [str] + S3 bucket name. + + Returns + ------- + client: [botocore.client.S3] + Amazon S3 client + """ + # AWS access / secret keys required + # s3 = boto3.resource('s3') + # bucket = s3.Bucket(era5_bucket) + + # No AWS keys required + client = boto3.client( + "s3", config=botocore.client.Config(signature_version=botocore.UNSIGNED) + ) + return client + + def get_catalog(self): + """return the catalog.""" + return { + "precipitation": { + "descriptions": "rainfall [mm/temporal_resolution]", + "units": "mm/temporal_resolution", + "temporal resolution": ["daily", "monthly"], + "file name": "rainfall", + "var_name": "R", + "bucket_name": "precipitation_amount_1hour_Accumulation", + } + } + + def get_variable(self, var_name) -> Dict[str, str]: + """get the details of a specific variable.""" + return super().get_variable(var_name) + + def get_available_years(self, bucket: str = "era5-pds"): + """The ERA5 data is chunked into distinct NetCDF files per variable, each containing a month of hourly data. These files are organized in the S3 bucket by year, month, and variable name. + + The data is structured as follows: + + /{year}/{month}/main.nc + /data/{var1}.nc + /{var2}.nc + /{....}.nc + /{varN}.nc + + - where year is expressed as four digits (e.g. YYYY) and month as two digits (e.g. MM). + + Parameters + ---------- + bucket: [str] + S3 bucket name + + Returns + ------- + List: + list of years that have available data. + """ + paginator = self.client.get_paginator("list_objects") + result = paginator.paginate(Bucket=bucket, Delimiter="/") + # for prefix in result.search('CommonPrefixes'): + # print(prefix.get('Prefix')) + years = [i.get("Prefix")[:-1] for i in result.search("CommonPrefixes")] + return years + + def get_available_data( + self, + date: str, + bucket: str = "era5-pds", + fmt: str = "%Y-%m-%d", + absolute_path: bool = False, + ) -> List[str]: + """get the available data at a given year. + + - Granule variable structure and metadata attributes are stored in main.nc. This file contains coordinate and + auxiliary variable data. This file is also annotated using NetCDF CF metadata conventions. + + Parameters + ---------- + date: [str] + date i.e. "YYYY-mm-dd" + bucket: [str] + The bucket you want to get its available data. Default is 'era5-pds'. + fmt: [str] + Date format. Default is "%Y-%m-%d". + absolute_path: [bool] + True if you want to get the file names including the whole path inside the bucket. + Default is False. + >>> absolute_path = True + [ + '2022/05/air_pressure_at_mean_sea_level.nc', + '2022/05/air_temperature_at_2_metres.nc', + '2022/05/air_temperature_at_2_metres_1hour_Maximum.nc', + '2022/05/air_temperature_at_2_metres_1hour_Minimum.nc', + '2022/05/dew_point_temperature_at_2_metres.nc', + '2022/05/eastward_wind_at_100_metres.nc' + ] + >>> absolute_path = False + [ + 'air_pressure_at_mean_sea_level.nc', + 'air_temperature_at_2_metres.nc', + 'air_temperature_at_2_metres_1hour_Maximum.nc', + 'air_temperature_at_2_metres_1hour_Minimum.nc', + 'dew_point_temperature_at_2_metres.nc', + 'eastward_wind_at_100_metres.nc' + ] + Returns + ------- + List: + available data in a list + """ + date_obj = dt.datetime.strptime(date, fmt) + # date = dt.date(2022,5,1) # update to desired date + prefix = date_obj.strftime("%Y/%m/") + response = self.client.list_objects_v2(Bucket=bucket, Prefix=prefix) + keys = S3.parse_response_metadata(response) + if absolute_path: + available_date = keys + else: + available_date = [i.split("/")[-1] for i in keys] + return available_date diff --git a/earth2observe/utils.py b/earth2observe/utils.py deleted file mode 100644 index a3697a4..0000000 --- a/earth2observe/utils.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import sys - - -def print_progress_bar( - i: int, - total: int, - prefix: str = "", - suffix: str = "", - decimals: int = 1, - length: int = 100, - fill: str = "█", -): - """print_progress_bar. - - Parameters - ---------- - i: [int] - Iteration number - total: [int] - Total iterations - prefix: [str] - Name after bar - suffix: [str] - Decimals of percentage - decimals: [int] - - length: [int] - width of the waitbar - fill: [str] - bar fill - """ - # Adjust when it is a linux computer - if os.name == "posix" and total == 0: - total = 0.0001 - - percent = ("{0:." + str(decimals) + "f}").format(100 * (i / float(total))) - filled = int(length * i // total) - bar = fill * filled + "-" * (length - filled) - - sys.stdout.write("\r%s |%s| %s%% %s" % (prefix, bar, percent, suffix)) - sys.stdout.flush() - - if i == total: - print() diff --git a/environment.yml b/environment.yml index 0f5ecb0..b0ce608 100644 --- a/environment.yml +++ b/environment.yml @@ -9,8 +9,10 @@ dependencies: - pandas >=1.4.4 - ecmwf-api-client >=1.6.3 - earthengine-api >=0.1.324 + - boto3 >=1.26.50 - joblib >=1.2.0 - - pyramids >=0.2.12 + - pyramids >=0.3.1 + - serapeum_utils >=0.1.1 - loguru >=0.6.0 - PyYAML >=6.0 - pathlib >=1.0.1 diff --git a/examples/abstract_implementation.py b/examples/abstract_implementation.py index b260695..0eda78c 100644 --- a/examples/abstract_implementation.py +++ b/examples/abstract_implementation.py @@ -1,11 +1,13 @@ from earth2observe.earth2observe import Earth2Observe -source = "chirps" +# unified parameters for all data sources. start = "2009-01-01" end = "2009-01-10" temporal_resolution = "daily" latlim = [4.19, 4.64] lonlim = [-75.65, -74.73] +#%% +source = "chirps" path = r"examples\data\chirps" variables = ["precipitation"] e2o = Earth2Observe( @@ -37,7 +39,7 @@ path = r"examples\data\ecmwf" source = "ecmwf" -variables = ["E"] +variables = ["precipitation"] e2o = Earth2Observe( data_source=source, start=start, @@ -48,4 +50,20 @@ temporal_resolution=temporal_resolution, path=path, ) +# e2o.download() + +#%% +path = r"examples\data\s3-backend" +source = "amazon-s3" +variables = ["precipitation"] +e2o = Earth2Observe( + data_source=source, + start=start, + end=end, + variables=variables, + # lat_lim=latlim, + # lon_lim=lonlim, + temporal_resolution=temporal_resolution, + path=path, +) e2o.download() diff --git a/examples/chirps_data.py b/examples/chirps_data.py index d717b82..84dc751 100644 --- a/examples/chirps_data.py +++ b/examples/chirps_data.py @@ -1,13 +1,16 @@ -from earth2observe.chirps import CHIRPS +from earth2observe.chirps import CHIRPS, Catalog +#%% +chirps_catalog = Catalog() +print(chirps_catalog.catalog) # %% precipitation start = "2009-01-01" -end = "2009-01-10" +end = "2009-01-2" time = "daily" latlim = [4.19, 4.64] lonlim = [-75.65, -74.73] -path = r"examples\data\chirps" +path = r"examples/data/delete/chirps" Coello = CHIRPS( start=start, end=end, diff --git a/examples/data/s3/.gitignore b/examples/data/s3/.gitignore new file mode 100644 index 0000000..f2c3201 --- /dev/null +++ b/examples/data/s3/.gitignore @@ -0,0 +1,7 @@ +# Ignore everything in this directory + +* + +# Except this file + +!.gitignore diff --git a/examples/ecmwf_data.py b/examples/ecmwf_data.py index d9979e4..fb16031 100644 --- a/examples/ecmwf_data.py +++ b/examples/ecmwf_data.py @@ -9,7 +9,7 @@ from earth2observe.ecmwf import ECMWF, Catalog rpath = os.getcwd() -path = rf"{rpath}\examples\data\ecmwf" +path = rf"{rpath}\delete\data\ecmwf" #%% precipitation start = "2009-01-01" end = "2009-01-10" @@ -19,8 +19,10 @@ # Temperature, Evapotranspiration variables = ["T", "E"] #%% -Vars = Catalog() -print(Vars.catalog) +var = "T" +catalog = Catalog() +print(catalog.catalog) +catalog.get_variable(var) #%% Temperature start = "2009-01-01" end = "2009-02-01" diff --git a/examples/s3_er5_data.py b/examples/s3_er5_data.py new file mode 100644 index 0000000..2d9e81f --- /dev/null +++ b/examples/s3_er5_data.py @@ -0,0 +1,34 @@ +import os + +from earth2observe.s3 import S3, Catalog + +#%% +s3_catalog = Catalog() +print(s3_catalog.catalog) +s3_catalog.get_variable("precipitation") +years = s3_catalog.get_available_years() +date = "2022-05-01" +# available_date_abs_path = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=True) +# available_date = s3_catalog.get_available_data(date, bucket='era5-pds', absolute_path=False) +#%% +start = "2022-05-01" +end = "2022-05-01" +time = "monthly" +lat = [4.190755, 4.643963] +lon = [-75.649243, -74.727286] +variables = ["precipitation"] +rpath = os.getcwd() +path = rf"{rpath}/examples/data/s3-era5" + +s3_era5 = S3( + temporal_resolution=time, + start=start, + end=end, + path=path, + variables=variables, + # lat_lim=lat, + # lon_lim=lon, +) +#%% +s3_era5.download() +#%% diff --git a/requirements.txt b/requirements.txt index 6c4cffa..8600d6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +boto3 >=1.26.50 earthengine-api >=0.1.324 ecmwf-api-client >=1.6.3 gdal >=3.5.3 @@ -8,6 +9,7 @@ numpy ==1.24.1 pandas >=1.4.4 pathlib >=1.0.1 pip >=22.3.1 -pyramids-gis >=0.2.12 +pyramids-gis >=0.3.1 PyYAML >=6.0 requests >=2.28.1 +serapeum_utils >=0.1.1 diff --git a/setup.py b/setup.py index 2fa0ea2..bce4d80 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name="earth2observe", - version="0.2.0", + version="0.2.1", description="remote sensing package", author="Mostafa Farrag", author_email="moah.farag@gmail.come", diff --git a/tests/conftest.py b/tests/conftest.py index 26b5b3c..ae9466a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,5 @@ +import os from pathlib import Path -from typing import List import pytest @@ -8,12 +8,17 @@ @pytest.fixture(scope="session") def dates() -> List: - return ["2009-01-01", "2009-01-05"] + return ["2009-01-01", "2009-01-02"] + + +@pytest.fixture(scope="session") +def monthly_dates() -> List: + return ["2009-01-01", "2009-02-01"] @pytest.fixture(scope="session") def number_downloaded_files() -> int: - return 5 + return 2 @pytest.fixture(scope="session") @@ -22,23 +27,42 @@ def daily_temporal_resolution() -> str: @pytest.fixture(scope="session") +def monthly_temporal_resolution() -> str: + return "monthly" + + +@pytest.fixture(scope="module") def lat_bounds() -> List: return [4.19, 4.64] -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def lon_bounds() -> List: return [-75.65, -74.73] -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def chirps_base_dir() -> str: - return "tests/data/delete/chirps" + rpath = Path(f"tests/data/delete/chirps") + if not os.path.exists(rpath): + os.makedirs(rpath) + return Path(rpath).absolute() -@pytest.fixture(scope="session") -def ecmwf_base_dir() -> Path: - return Path("tests/data/delete/ecmwf").absolute() +@pytest.fixture(scope="module") +def ecmwf_base_dir() -> str: + path = "tests/data/delete/ecmwf" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + + +@pytest.fixture(scope="module") +def s3_era5_base_dir() -> str: + path = "tests/data/delete/s3-era5" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() @pytest.fixture(scope="session") @@ -52,13 +76,13 @@ def ecmwf_variables() -> List[str]: @pytest.fixture(scope="session") -def ecmwf_data_source() -> str: - return "ecmwf" +def s3_era5_variables() -> List[str]: + return ["precipitation"] @pytest.fixture(scope="session") -def ecmwf_data_source_output_dir() -> str: - return Path("tests/data/delete/ecmwf-backend").absolute() +def ecmwf_data_source() -> str: + return "ecmwf" @pytest.fixture(scope="session") @@ -67,5 +91,29 @@ def chirps_data_source() -> str: @pytest.fixture(scope="session") +def s3_data_source() -> str: + return "amazon-s3" + + +@pytest.fixture(scope="module") +def ecmwf_data_source_output_dir() -> str: + path = "tests/data/delete/ecmwf-backend" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + + +@pytest.fixture(scope="module") def chirps_data_source_output_dir() -> str: - return Path("tests/data/delete/chirps-backend").absolute() + path = "tests/data/delete/chirps-backend" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() + + +@pytest.fixture(scope="module") +def s3_era5_data_source_output_dir() -> str: + path = "tests/data/delete/s3-era5-backend" + if not os.path.exists(path): + os.makedirs(path) + return Path(path).absolute() diff --git a/tests/test_chirps.py b/tests/test_chirps.py index ff09f81..5f6f2fe 100644 --- a/tests/test_chirps.py +++ b/tests/test_chirps.py @@ -8,7 +8,7 @@ from earth2observe.chirps import CHIRPS -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def test_create_chirps_object( dates: List, daily_temporal_resolution: str, @@ -43,9 +43,10 @@ def test_download( fname = test_create_chirps_object.clipped_fname test_create_chirps_object.download() - filelist = glob.glob( - os.path.join(f"{chirps_base_dir}/chirps/precipitation", f"{fname}*.tif") - ) + filelist = glob.glob(os.path.join(f"{chirps_base_dir}", f"{fname}*.tif")) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{chirps_base_dir}/chirps/precipitation") + try: + shutil.rmtree(f"{chirps_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted") diff --git a/tests/test_earth2observe.py b/tests/test_earth2observe.py index 44656c7..fb10850 100644 --- a/tests/test_earth2observe.py +++ b/tests/test_earth2observe.py @@ -8,10 +8,11 @@ from earth2observe.chirps import CHIRPS from earth2observe.earth2observe import Earth2Observe from earth2observe.ecmwf import ECMWF +from earth2observe.s3 import S3 class TestChirpsBackend: - @pytest.fixture(scope="session") + @pytest.fixture(scope="module") def test_chirps_data_source_instantiate_object( self, chirps_data_source: str, @@ -47,17 +48,18 @@ def test_download_chirps_backend( test_chirps_data_source_instantiate_object.download() fname = "P_CHIRPS" filelist = glob.glob( - os.path.join( - f"{chirps_data_source_output_dir}/chirps/precipitation", f"{fname}*.tif" - ) + os.path.join(f"{chirps_data_source_output_dir}", f"{fname}*.tif") ) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{chirps_data_source_output_dir}/chirps/precipitation") + try: + shutil.rmtree(f"{chirps_data_source_output_dir}") + except PermissionError: + print("the downloaded files could not be deleted") class TestECMWFBackend: - @pytest.fixture(scope="session") + @pytest.fixture(scope="module") def test_ecmwf_data_source_instantiate_object( self, ecmwf_data_source: str, @@ -83,16 +85,60 @@ def test_ecmwf_data_source_instantiate_object( assert e2o.datasource.vars == ecmwf_variables return e2o - def test_download_chirps_backend( + def test_download_ecmwf_backend( self, test_ecmwf_data_source_instantiate_object: CHIRPS, ecmwf_data_source_output_dir: str, number_downloaded_files: int, ): test_ecmwf_data_source_instantiate_object.download() - filelist = glob.glob( - os.path.join(f"{ecmwf_data_source_output_dir}/daily/Evaporation/", f"*.tif") + filelist = glob.glob(os.path.join(f"{ecmwf_data_source_output_dir}", f"*.tif")) + assert len(filelist) == number_downloaded_files + # delete the files + try: + shutil.rmtree(f"{ecmwf_data_source_output_dir}") + except PermissionError: + print("the downloaded files could not be deleted") + + +class TestS3Backend: + @pytest.fixture(scope="module") + def test_s3_data_source_instantiate_object( + self, + s3_data_source: str, + monthly_dates: List, + monthly_temporal_resolution: str, + s3_era5_variables: List[str], + lat_bounds: List, + lon_bounds: List, + s3_era5_data_source_output_dir: str, + ): + e2o = Earth2Observe( + data_source=s3_data_source, + start=monthly_dates[0], + end=monthly_dates[1], + variables=s3_era5_variables, + lat_lim=lat_bounds, + lon_lim=lon_bounds, + temporal_resolution=monthly_temporal_resolution, + path=s3_era5_data_source_output_dir, ) + assert isinstance(e2o.DataSources, dict) + assert isinstance(e2o.datasource, S3) + assert e2o.datasource.vars == s3_era5_variables + return e2o + + def test_download_s3_backend( + self, + test_s3_data_source_instantiate_object: S3, + s3_era5_data_source_output_dir: str, + number_downloaded_files: int, + ): + test_s3_data_source_instantiate_object.download() + filelist = glob.glob(os.path.join(f"{s3_era5_data_source_output_dir}", f"*.nc")) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{ecmwf_data_source_output_dir}/daily") + try: + shutil.rmtree(f"{s3_era5_data_source_output_dir}") + except PermissionError: + print("the downloaded files could not be deleted") diff --git a/tests/test_ecmwf.py b/tests/test_ecmwf.py index a1d1c95..bbacc66 100644 --- a/tests/test_ecmwf.py +++ b/tests/test_ecmwf.py @@ -8,7 +8,7 @@ from earth2observe.ecmwf import ECMWF -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def test_create_ecmwf_object( dates: List, lat_bounds: List, @@ -34,7 +34,10 @@ def test_download( number_downloaded_files: int, ): test_create_ecmwf_object.download() - filelist = glob.glob(os.path.join(f"{ecmwf_base_dir}/daily/Evaporation/", f"*.tif")) + filelist = glob.glob(os.path.join(f"{ecmwf_base_dir}", f"*.tif")) assert len(filelist) == number_downloaded_files # delete the files - shutil.rmtree(f"{ecmwf_base_dir}/daily") + try: + shutil.rmtree(f"{ecmwf_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted") diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 0000000..dd3d930 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,43 @@ +import glob +import os +import shutil +from typing import List + +import pytest + +from earth2observe.s3 import S3 + + +@pytest.fixture(scope="module") +def test_create_s3_object( + monthly_dates: List, + lat_bounds: List, + lon_bounds: List, + s3_era5_base_dir: str, + s3_era5_variables: List[str], +): + Coello = S3( + start=monthly_dates[0], + end=monthly_dates[1], + lat_lim=lat_bounds, + lon_lim=lon_bounds, + path=s3_era5_base_dir, + variables=s3_era5_variables, + ) + assert isinstance(Coello, S3) + return Coello + + +def test_download( + test_create_s3_object: S3, + s3_era5_base_dir: str, + number_downloaded_files: int, +): + test_create_s3_object.download() + filelist = glob.glob(os.path.join(f"{s3_era5_base_dir}", f"*.nc")) + assert len(filelist) == number_downloaded_files + # delete the files + try: + shutil.rmtree(f"{s3_era5_base_dir}") + except PermissionError: + print("the downloaded files could not be deleted")