From e0d38bb15b3c7cb69428cf0b6aa7484b76772151 Mon Sep 17 00:00:00 2001 From: "Hong-Kit \"Randy\" Yeung" Date: Wed, 13 Sep 2023 13:51:06 -0500 Subject: [PATCH] Allow CNM to Granule to process `http` addresses (#206) * allowing cmn to granule to process http also * fix build.gradle issue so it works with github actions * change gradle version from 4 to 8 --- .github/workflows/build-pipeline.yml | 2 +- build.gradle | 2 +- pom.xml | 2 +- .../gov/nasa/cumulus/CnmToGranuleHandler.java | 4 +- .../nasa/cumulus/CnmToGranuleHandlerTest.java | 16 ++++ src/test/resources/http_input.json | 88 +++++++++++++++++++ src/test/resources/http_output.json | 43 +++++++++ 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/http_input.json create mode 100644 src/test/resources/http_output.json diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml index 6f7363e..88372cc 100755 --- a/.github/workflows/build-pipeline.yml +++ b/.github/workflows/build-pipeline.yml @@ -27,7 +27,7 @@ jobs: java-version: '8.0.232' - uses: gradle/gradle-build-action@v1 with: - gradle-version: 4.0.1 + gradle-version: 8.0.1 - name: Install Python 3 uses: actions/setup-python@v2 with: diff --git a/build.gradle b/build.gradle index 5e9673b..6e41fc5 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ task buildZip(type: Zip) { into('lib') { from configurations.runtimeClasspath } - archiveName 'cnmToGranule.zip' + archiveFileName.set('cnmToGranule.zip') } build.dependsOn buildZip diff --git a/pom.xml b/pom.xml index 694abab..487f085 100755 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.2.2 + 2.13.4.2 diff --git a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java index 8e9ba7a..f2adac5 100644 --- a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java +++ b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java @@ -151,7 +151,8 @@ public String PerformFunction(String input, Context context) throws Exception { String uri = StringUtils.trim(cnmFile.get("uri").getAsString()); if (StringUtils.beginsWithIgnoreCase(uri, "s3://")) { granuleFile = buildS3GranuleFile(cnmFile); - } else if (StringUtils.beginsWithIgnoreCase(uri, "https://")) { + } else if (StringUtils.beginsWithIgnoreCase(uri, "https://") || + StringUtils.beginsWithIgnoreCase(uri, "http://")) { granuleFile = buildHttpsGranuleFile(cnmFile); } else if (StringUtils.beginsWithIgnoreCase(uri, "sftp://")) { granuleFile = buildSftpGranuleFile(cnmFile); @@ -214,6 +215,7 @@ public JsonObject buildHttpsGranuleFile(JsonObject cnmFile){ String uri = StringUtils.trim(cnmFile.get("uri").getAsString()); AdapterLogger.LogInfo(this.className + " uri: " + uri); String path = uri.replace("https://", ""); + path = path.replace("http://", ""); // Work-a-around to working with http links also //find the path by getting character from (first / plus 1) to lastIndex of / String url_path = path.substring(path.indexOf("/") + 1, path.lastIndexOf("/")); diff --git a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java index f2e731c..224f27f 100644 --- a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java +++ b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java @@ -147,4 +147,20 @@ public void testBuildSftpGranuleFile() throws Exception { assert(expectedJson.equals(outputJson.getAsJsonObject("output"))); } + + public void testBuildHttpGranuleFile() throws Exception { + ClassLoader classLoader = getClass().getClassLoader(); + File inputJsonFile = new File(classLoader.getResource("http_input.json").getFile()); + File expectedJsonFile = new File(classLoader.getResource("http_output.json").getFile()); + + String input = new String(Files.readAllBytes(inputJsonFile.toPath())); + String expected = new String(Files.readAllBytes(expectedJsonFile.toPath())); + JsonObject expectedJson = new JsonParser().parse(expected).getAsJsonObject(); + + CnmToGranuleHandler cnmToGranuleHandler = new CnmToGranuleHandler(); + String output = cnmToGranuleHandler.PerformFunction(input, null); + JsonObject outputJson = new JsonParser().parse(output).getAsJsonObject(); + + assert(expectedJson.equals(outputJson.getAsJsonObject("output"))); + } } diff --git a/src/test/resources/http_input.json b/src/test/resources/http_input.json new file mode 100644 index 0000000..40d8208 --- /dev/null +++ b/src/test/resources/http_input.json @@ -0,0 +1,88 @@ +{ + "input": { + "collection": "HLSL30", + "identifier": "2ffc5609-6ab4-4f58-b422-4d71ff43b269", + "duplicationid": "HLS.L30.T11VNK.2020259T185742.v2.0", + "version": "1.4", + "submissionTime": "2021-10-29T19:47:06Z", + "product": { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0", + "dataVersion": "2.0", + "id": "HLS.L30.T11VNK.2020259T185742.v2.0", + "files": [ + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "size": 6430, + "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "type": "metadata" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "size": 798505, + "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "size": 561155, + "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "size": 194484, + "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3", + "checksumType": "SHA512", + "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "type": "browse" + } + ] + } + }, + "config": { + "collection": { + "files": [ + { + "regex": ".*.h5$", + "sampleFileName": "HLSL30_product_0001-of-0050.h5", + "type": "data", + "bucket": "protected" + }, + { + "regex": ".*.iso.xml$", + "sampleFileName": "HLSL30_product_0001-of-0019.iso.xml", + "type": "metadata", + "bucket": "protected" + }, + { + "regex": ".*.cmr.json$", + "sampleFileName": "HLSL30_product_0001-of-0019.cmr.json", + "type": "metadata", + "bucket": "public" + } + ], + "name": "HLSL30", + "granuleIdExtraction": "^(.*)((\\.cmr\\.json)|(\\.h5)|(\\.h5\\.mp))$", + "granuleId": "^.*$", + "dataType": "HLSL30", + "provider_path": "HLSL30/", + "version": "2.0", + "updatedAt": 1552434051881, + "duplicateHandling": "replace", + "sampleFileName": "HLSL30_product_0001-of-0050.h5", + "createdAt": 1552434051881, + "meta": { + "required-files": [ + ".*.h5$", + ".*.iso.xml$" + ] + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/http_output.json b/src/test/resources/http_output.json new file mode 100644 index 0000000..abf0655 --- /dev/null +++ b/src/test/resources/http_output.json @@ -0,0 +1,43 @@ +{ + "granules": [ + { + "granuleId": "HLS.L30.T11VNK.2020259T185742.v2.0", + "version": "2.0", + "dataType": "HLSL30", + "files": [ + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 6430, + "checksumType": "SHA512", + "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3", + "type": "metadata" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 798505, + "checksumType": "SHA512", + "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 561155, + "checksumType": "SHA512", + "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678", + "type": "data" + }, + { + "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg", + "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0", + "size": 194484, + "checksumType": "SHA512", + "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3", + "type": "browse" + } + ] + } + ] +} \ No newline at end of file