From 5c255e40a2ca33aa058aa6cc1737bb6084d84168 Mon Sep 17 00:00:00 2001 From: zoewalschots Date: Wed, 21 Jun 2023 22:51:40 +0000 Subject: [PATCH] Edited OPERA Cloud Notebook Added earthaccess to simplify authentication, search, and access --- notebooks/datasets/OPERA_GIS_Cloud.ipynb | 454 +++++++++++------------ 1 file changed, 216 insertions(+), 238 deletions(-) diff --git a/notebooks/datasets/OPERA_GIS_Cloud.ipynb b/notebooks/datasets/OPERA_GIS_Cloud.ipynb index b017a1d1..ef5aaa97 100644 --- a/notebooks/datasets/OPERA_GIS_Cloud.ipynb +++ b/notebooks/datasets/OPERA_GIS_Cloud.ipynb @@ -12,7 +12,7 @@ "\n", "## Summary & Learning Objectives\n", "#### Notebook showcasing how to work with OPERA DSWx data in the cloud\n", - "- Utilizing the Common Metadata Repository (CMR) to search for the data then using the cloud to open the dataset.\n", + "- Utilizing the earthaccess Python package. For more information visit: https://nsidc.github.io/earthaccess/\n", "- Option to query the new dataset based on users choice; either by classified layer 'B01' or sensor ('L8_30_v1.0_B01_WTR'), etc.\n", "- Visualizing the dataset based on its classified layer values.\n", "- Mosaicking multiple layers into a single layer.\n", @@ -22,16 +22,12 @@ "### 1. Compute environment \n", "\n", "This tutorial can only be run in the following environment:\n", - "- **AWS instance running in us-west-2**: NASA Earthdata Cloud data in S3 can be directly accessed via temporary credentials; this access is limited to requests made within the US West (Oregon) (code: `us-west-2`) AWS region.\n", + "- **AWS instance running in us-west-2**: NASA Earthdata Cloud data in S3 can be directly accessed via and s3fs session; this access is limited to requests made within the US West (Oregon) (code: `us-west-2`) AWS region.\n", "\n", "### 2. Earthdata Login\n", "\n", "An Earthdata Login account is required to access data, as well as discover restricted data, from the NASA Earthdata system. Thus, to access NASA data, you need Earthdata Login. Please visit https://urs.earthdata.nasa.gov to register and manage your Earthdata Login account. This account is free to create and only takes a moment to set up.\n", "\n", - "### 3. netrc File\n", - "\n", - "You will need a `.netrc` file containing your NASA Earthdata Login credentials in order to execute the notebooks. A `.netrc` file can be created manually within text editor and saved to your home directory. For additional information see: [Authentication for NASA Earthdata tutorial](https://nasa-openscapes.github.io/2021-Cloud-Workshop-AGU/tutorials/02_NASA_Earthdata_Authentication.html).\n", - "\n", "### Import Packages" ] }, @@ -62,137 +58,56 @@ "import numpy as np\n", "from pathlib import Path\n", "from urllib.request import urlretrieve\n", - "from json import dumps" + "from json import dumps\n", + "import earthaccess\n", + "from earthaccess import Auth, DataCollections, DataGranules, Store" ] }, { "cell_type": "markdown", "id": "40db2712-e298-4b6a-9e85-b5a97d91a8ad", - "metadata": {}, - "source": [ - "## Authentication with netrc\n", - "\n", - "In this notebook, we will be calling the authentication in the below cell, a work around if you do not yet have a netrc file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "baa5b0e8-ac3d-431f-842d-20478c254857", "metadata": { "tags": [] }, - "outputs": [], "source": [ - "from urllib import request\n", - "from http.cookiejar import CookieJar\n", - "from getpass import getpass\n", - "import netrc\n", - "from platform import system\n", - "from os.path import join, isfile, basename, abspath, expanduser\n", - "\n", - "def setup_earthdata_login_auth(endpoint: str='urs.earthdata.nasa.gov'):\n", - " netrc_name = \"_netrc\" if system()==\"Windows\" else \".netrc\"\n", - " try:\n", - " username, _, password = netrc(file=join(expanduser('~'), netrc_name)).authenticators(endpoint)\n", - " except (FileNotFoundError, TypeError):\n", - " print('Please provide your Earthdata Login credentials for access.')\n", - " print('Your info will only be passed to %s and will not be exposed in Jupyter.' % (endpoint))\n", - " username = input('Username: ')\n", - " password = getpass('Password: ')\n", - " manager = request.HTTPPasswordMgrWithDefaultRealm()\n", - " manager.add_password(None, endpoint, username, password)\n", - " auth = request.HTTPBasicAuthHandler(manager)\n", - " jar = CookieJar()\n", - " processor = request.HTTPCookieProcessor(jar)\n", - " opener = request.build_opener(auth, processor)\n", - " request.install_opener(opener)\n", - " \n", - "setup_earthdata_login_auth('urs.earthdata.nasa.gov')" - ] - }, - { - "cell_type": "markdown", - "id": "2fbf69b7-6574-4ec4-b3da-4a3b3d3c290e", - "metadata": {}, - "source": [ - "### Get Temporary AWS Credentials for Access\n", - "S3 is an 'object store' hosted in AWS for cloud processing. Direct S3 access is achieved by passing NASA supplied temporary credentials to AWS so we can interact with S3 objects from applicable Earthdata Cloud buckets. Note, these temporary credentials are valid for only 1 hour. A netrc file is required to aquire these credentials. Use the [NASA Earthdata Authentication](https://github.com/NASA-Openscapes/2021-Cloud-Hackathon/blob/main/tutorials/04_NASA_Earthdata_Authentication.ipynb) to create a netrc file in your home directory. (Note: A NASA Earthdata Login is required to access data from the NASA Earthdata system. Please visit https://urs.earthdata.nasa.gov to register and manage your Earthdata Login account. This account is free to create and only takes a moment to set up.)\n", + "## Authentication with `earthaccess`\n", "\n", - "The following crediential is for PODAAC, but other credentials are needed to access data from other NASA DAACs." + "In this notebook, we will be calling the authentication in the below cell." ] }, { "cell_type": "code", - "execution_count": 8, - "id": "f1135972-d61c-49fc-90fe-d0a25fec9f7c", + "execution_count": null, + "id": "b2e86921-fdea-4cc4-8f39-bb31d1419ec3", "metadata": { "tags": [] }, "outputs": [], "source": [ - "s3_cred_endpoint = {\n", - " 'podaac':'https://archive.podaac.earthdata.nasa.gov/s3credentials'\n", - "}" + "auth = earthaccess.login(strategy=\"interactive\", persist=True)" ] }, { "cell_type": "markdown", - "id": "0c5265ba-8fdd-44f8-b9a0-1a9841966867", - "metadata": {}, - "source": [ - "Create a function to make a request to an endpoint for temporary credentials." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "f4aa27df-d644-4d26-bf8f-f670de0200dd", + "id": "acaeb5e5-6bd4-4842-ae4f-40c2ab497e62", "metadata": { "tags": [] }, - "outputs": [], "source": [ - "def get_temp_creds(provider):\n", - " return requests.get(s3_cred_endpoint[provider]).json()" + "## Set up an s3fs session for Direct Access\n", + "s3fs sessions are used for authenticated access to s3 bucket and allows for typical file-system style operations. Below we create session by passing in the data provider." ] }, { "cell_type": "code", - "execution_count": 10, - "id": "8c19c7b1-5fbe-46df-be55-8ec4cc641d5f", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "temp_creds_req = get_temp_creds('podaac')\n", - "# temp_creds_req" - ] - }, - { - "cell_type": "markdown", - "id": "acaeb5e5-6bd4-4842-ae4f-40c2ab497e62", - "metadata": {}, - "source": [ - "### Set up an s3fs session for Direct Access\n", - "s3fs sessions are used for authenticated access to s3 bucket and allows for typical file-system style operations. Below we create session by passing in the temporary credentials we recieved from our temporary credentials endpoint." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "a70206cd-a959-4c45-a571-ec211cd3a941", + "execution_count": 3, + "id": "53bf8ed3-5588-42b3-b4bd-2c80bfc293cd", "metadata": { "tags": [] }, "outputs": [], "source": [ - "fs_s3 = s3fs.S3FileSystem(anon=False, \n", - " key=temp_creds_req['accessKeyId'], \n", - " secret=temp_creds_req['secretAccessKey'], \n", - " token=temp_creds_req['sessionToken'],\n", - " client_kwargs={'region_name':'us-west-2'})" + "fs_s3 = Store(auth).get_s3fs_session(daac=\"PODAAC\", provider=\"POCLOUD\")" ] }, { @@ -200,7 +115,7 @@ "id": "bb35670e-7e5a-4569-83b7-2ab3d28bd3a1", "metadata": {}, "source": [ - "## Search Common Metadata Repository (CMR) for OPERA DSWx\n", + "## Search Using `earthaccess` for OPERA DSWx\n", "Each dataset has it's own unique collection ID. For the OPERA_L3_DSWX-HLS_PROVISIONAL_V1 dataset, we can find the collection ID [here](https://podaac.jpl.nasa.gov/dataset/OPERA_L3_DSWX-HLS_PROVISIONAL_V1).\n", "\n", "For this tutorial, we are looking at the Lake Powell Reservoir.\n", @@ -212,8 +127,8 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "e87311e7-6651-4b3c-9fa2-3176acc79dae", + "execution_count": 4, + "id": "93198552-2adc-4655-9d98-6222c7d0dfc1", "metadata": { "tags": [] }, @@ -222,104 +137,90 @@ "name": "stdout", "output_type": "stream", "text": [ - "50\n" + "Granule hits: 50\n" ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# the URL of the CMR service\n", - "cmr_url = 'https://cmr.earthdata.nasa.gov/search/granules.json'\n", - "\n", - "#used to define parameters such as the concept-id, temporal and bounding box searches\n", - "parameters = {'collection_concept_id':'C2617126679-POCLOUD',\n", - " 'temporal': '2023-04-11T00:00:00Z,2023-05-02T23:59:59Z',\n", - " 'bounding_box':\"-111.144811,36.980121,-110.250799,37.915625\", #could also use a bounding box\n", - " 'page_size':2000} #default will only return 10 granules, so we set it to the max\n", - "\n", - "#request the granules from this collection\n", - "response = requests.post(cmr_url, params=parameters)\n", - "\n", - "if len(response.json()['feed']['entry'])>0:\n", - " print(len(response.json()['feed']['entry'])) #print out number of files found\n", - " #print(dumps(response.json()['feed']['entry'][0], indent=2)) #print out the first file information" - ] - }, - { - "cell_type": "markdown", - "id": "1385be5f-1537-4d40-a7e5-a606589a04da", - "metadata": {}, - "source": [ - "There are 50 files between this time range, and for the two dates we are interested in, there are 4 files each within the bounding box region. The list of timestamps are printed below. We want 4/11/2023 and 5/2/2023." + "Query = DataGranules().concept_id(\"C2617126679-POCLOUD\").bounding_box(-111.144811,36.980121,-110.250799,37.915625).temporal(\"2023-04-11T00:00:00\",\"2023-05-02T23:59:59\")\n", + "print(f\"Granule hits: {Query.hits()}\")\n", + "cloud_granules = Query.get()\n", + "# is this a cloud hosted data granule?\n", + "cloud_granules[0].cloud_hosted" ] }, { "cell_type": "code", - "execution_count": 4, - "id": "24604dbe-440a-4a24-ab16-db18b6caeaf1", + "execution_count": 5, + "id": "f4b73d49-05cb-486a-a3ed-f22f6a76d817", "metadata": { "tags": [] }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-04-11T18:02:22.780Z\n", - "2023-04-11T18:02:22.780Z\n", - "2023-04-11T18:02:22.780Z\n", - "2023-04-11T18:02:22.780Z\n", - "2023-04-12T18:23:47.043Z\n", - "2023-04-12T18:23:51.075Z\n", - "2023-04-12T18:24:01.646Z\n", - "2023-04-12T18:24:05.522Z\n", - "2023-04-14T18:13:50.218Z\n", - "2023-04-14T18:13:54.875Z\n", - "2023-04-14T18:14:04.643Z\n", - "2023-04-14T18:14:07.852Z\n", - "2023-04-17T18:23:45.423Z\n", - "2023-04-17T18:23:49.451Z\n", - "2023-04-17T18:24:00.009Z\n", - "2023-04-17T18:24:03.899Z\n", - "2023-04-18T18:08:30.981Z\n", - "2023-04-18T18:08:30.981Z\n", - "2023-04-19T18:13:52.221Z\n", - "2023-04-19T18:13:56.951Z\n", - "2023-04-19T18:14:06.643Z\n", - "2023-04-19T18:14:09.873Z\n", - "2023-04-20T17:56:08.850Z\n", - "2023-04-20T17:56:08.850Z\n", - "2023-04-20T17:56:08.850Z\n", - "2023-04-20T17:56:08.850Z\n", - "2023-04-22T18:23:48.224Z\n", - "2023-04-22T18:23:52.246Z\n", - "2023-04-22T18:24:02.814Z\n", - "2023-04-22T18:24:06.701Z\n", - "2023-04-24T18:13:49.957Z\n", - "2023-04-24T18:13:54.676Z\n", - "2023-04-24T18:14:04.380Z\n", - "2023-04-24T18:14:07.610Z\n", - "2023-04-27T18:02:19.254Z\n", - "2023-04-27T18:02:19.254Z\n", - "2023-04-27T18:02:19.254Z\n", - "2023-04-27T18:02:19.254Z\n", - "2023-04-27T18:23:46.262Z\n", - "2023-04-27T18:23:50.297Z\n", - "2023-04-27T18:24:00.859Z\n", - "2023-04-27T18:24:04.747Z\n", - "2023-04-29T18:13:52.295Z\n", - "2023-04-29T18:13:57.054Z\n", - "2023-04-29T18:14:06.722Z\n", - "2023-04-29T18:14:09.959Z\n", - "2023-05-02T18:23:48.573Z\n", - "2023-05-02T18:23:52.603Z\n", - "2023-05-02T18:24:03.163Z\n", - "2023-05-02T18:24:07.046Z\n" - ] + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Data: OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B01_WTR.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B02_BWTR.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B03_CONF.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B04_DIAG.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B05_WTR-1.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B06_WTR-2.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B07_LAND.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B08_SHAD.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B09_CLOUD.tifOPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B10_DEM.tif

\n", + "

Size: 0 MB

\n", + "

Spatial: {'HorizontalSpatialDomain': {'Geometry': {'BoundingRectangles': [{'WestBoundingCoordinate': -111, 'SouthBoundingCoordinate': 37.04, 'EastBoundingCoordinate': -109.749, 'NorthBoundingCoordinate': 38.036}]}}}

\n", + "
\n", + "
\n", + " \"Data\n", + "
\n", + "
\n", + "
\n", + "
\n", + " " + ], + "text/plain": [ + "Collection: {'Version': '1.0', 'ShortName': 'OPERA_L3_DSWX-HLS_PROVISIONAL_V1'}\n", + "Spatial coverage: {'HorizontalSpatialDomain': {'Geometry': {'BoundingRectangles': [{'WestBoundingCoordinate': -111, 'SouthBoundingCoordinate': 37.04, 'EastBoundingCoordinate': -109.749, 'NorthBoundingCoordinate': 38.036}]}}}\n", + "Temporal coverage: {'RangeDateTime': {'EndingDateTime': '2023-04-11T18:02:22.780Z', 'BeginningDateTime': '2023-04-11T18:02:22.780Z'}}\n", + "Size(MB): 0\n", + "Data: ['https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B01_WTR.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B02_BWTR.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B03_CONF.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B04_DIAG.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B05_WTR-1.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B06_WTR-2.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B07_LAND.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B08_SHAD.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B09_CLOUD.tif', 'https://archive.podaac.earthdata.nasa.gov/podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B10_DEM.tif']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "for k in response.json()['feed']['entry']:\n", - " print(k['time_start'])" + "# Let's pretty print this\n", + "cloud_granules[0]" ] }, { @@ -327,7 +228,7 @@ "id": "63b64bf9-8d24-47f3-82f8-50320dd81fa7", "metadata": {}, "source": [ - "### Get S3 Bucket links from CMR search results\n", + "### Get S3 Bucket links from search results\n", "\n", "Because we are working within the AWS cloud, let's get the S3 bucket links for the 8 desired files. We want to query the dataset based on a specific classified layer 'B01' or sensor ('L8_30_v1.0_B01_WTR').\n", "\n", @@ -336,8 +237,42 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "3ea12324-e3d8-42ae-8fe7-a7d35e6e5089", + "execution_count": 6, + "id": "b84de729-ab41-4361-8bf2-b71d8c4d0fbd", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B01_WTR.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B02_BWTR.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B03_CONF.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B04_DIAG.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B05_WTR-1.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B06_WTR-2.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B07_LAND.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B08_SHAD.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B09_CLOUD.tif',\n", + " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B10_DEM.tif']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#extract S3 bucket links\n", + "data_links = [g.data_links(access=\"direct\") for g in cloud_granules]\n", + "data_links[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e2b72f1b-d940-4156-92fb-8c0f444edebf", "metadata": { "tags": [] }, @@ -351,66 +286,99 @@ } ], "source": [ - "S3 = []\n", - "for r in response.json()['feed']['entry']:\n", - " for l in r['links']:\n", - " if 's3://podaac-ops-cumulus-protected/' in l['href']:\n", - " if 'B01_WTR' in l['href']: #Specify which specific layer you would like to download. If not, comment out line to download the whole collection.\n", - " S3.append(l['href'])\n", - "\n", - "del S3[4:46] #remove the links to other dates not required for desired analysis\n", - "print(len(S3))" + "#add the S3 bucket links to a list, here we are looking for B01_WTR layer and two dates specified earlier\n", + "s3 = []\n", + "for r in data_links:\n", + " for l in r:\n", + " if 'B01_WTR' in l: \n", + " if '20230411' in l:\n", + " s3.append(l)\n", + " if '20230502' in l:\n", + " s3.append(l)\n", + "\n", + "print(len(s3))" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "23362378-24c7-4eb3-ac65-f0d74a42b516", + "execution_count": 9, + "id": "eede23b6-9753-43d4-97f2-966844f90eb1", "metadata": { "tags": [] }, "outputs": [ { "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4eed53d318434e5f8d1120c63ae034ec", + "version_major": 2, + "version_minor": 0 + }, "text/plain": [ - "['s3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230411T180222Z_20230414T030954Z_L8_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SVF_20230411T180222Z_20230414T030950Z_L8_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SVG_20230411T180222Z_20230414T030945Z_L8_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWF_20230411T180222Z_20230414T031011Z_L8_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWG_20230502T180919Z_20230504T155716Z_S2B_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SVG_20230502T180919Z_20230504T094859Z_S2B_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SWF_20230502T180919Z_20230504T155656Z_S2B_30_v1.0_B01_WTR.tif',\n", - " 's3://podaac-ops-cumulus-protected/OPERA_L3_DSWX-HLS_PROVISIONAL_V1/OPERA_L3_DSWx-HLS_T12SVF_20230502T180919Z_20230504T094852Z_S2B_30_v1.0_B01_WTR.tif']" + "SUBMITTING | : 0%| | 0/8 [00:00" + "" ] }, - "execution_count": 14, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -483,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "753c832e-6be5-4c49-b4f3-2b1cb03d7836", "metadata": { "tags": [] @@ -498,9 +466,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "7c862b33-6c47-4ac6-8e56-b79b8e0e4227", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -543,7 +513,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "07f58c9d-3879-4424-9f53-08199dd23f28", "metadata": { "tags": [] @@ -572,7 +542,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "ea887821-ee97-4c08-ada4-3c6d4defe788", "metadata": { "tags": [] @@ -596,7 +566,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "dc1bd1bb-f6ea-4707-a717-e9a2998c00a4", "metadata": { "tags": [] @@ -610,7 +580,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "4e4fdefd-51c3-4d56-a3ce-6db0283d9d6f", "metadata": { "tags": [] @@ -624,7 +594,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "id": "fc50b63d-f4b8-400e-99c3-b8ff334d70f6", "metadata": { "tags": [] @@ -686,7 +656,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "a55e49b2-5a30-4024-b448-9f4804802b26", "metadata": { "tags": [] @@ -750,6 +720,14 @@ "\n", "plt.show()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dfeacea-93b5-435c-9fcb-53b64bd5075a", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -768,7 +746,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.16" } }, "nbformat": 4,