Allow CNM to Granule to process http addresses (#206)

* allowing cmn to granule to process http also * fix build.gradle issue so it works with github actions * change gradle version from 4 to 8
podaac · Sep 13, 2023 · e0d38bb · e0d38bb
1 parent c0a8c93
commit e0d38bb
Show file tree

Hide file tree

Showing 7 changed files with 153 additions and 4 deletions.
diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml
@@ -27,7 +27,7 @@ jobs:
           java-version: '8.0.232'
       - uses: gradle/gradle-build-action@v1
         with:
-          gradle-version: 4.0.1
+          gradle-version: 8.0.1
       - name: Install Python 3
         uses: actions/setup-python@v2
         with:

diff --git a/build.gradle b/build.gradle
@@ -13,7 +13,7 @@ task buildZip(type: Zip) {
     into('lib') {
         from configurations.runtimeClasspath
     }
-    archiveName 'cnmToGranule.zip'
+    archiveFileName.set('cnmToGranule.zip')
 }
 
 build.dependsOn buildZip
diff --git a/pom.xml b/pom.xml
@@ -65,7 +65,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.2.2</version>
+            <version>2.13.4.2</version>
         </dependency>
     </dependencies>
 </project>
diff --git a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java
@@ -151,7 +151,8 @@ public String PerformFunction(String input, Context context) throws Exception {
             String uri = StringUtils.trim(cnmFile.get("uri").getAsString());
             if (StringUtils.beginsWithIgnoreCase(uri, "s3://")) {
                 granuleFile = buildS3GranuleFile(cnmFile);
-            } else if (StringUtils.beginsWithIgnoreCase(uri, "https://")) {
+            } else if (StringUtils.beginsWithIgnoreCase(uri, "https://") ||
+                    StringUtils.beginsWithIgnoreCase(uri, "http://")) {
                 granuleFile = buildHttpsGranuleFile(cnmFile);
             } else if (StringUtils.beginsWithIgnoreCase(uri, "sftp://")) {
                 granuleFile = buildSftpGranuleFile(cnmFile);
@@ -214,6 +215,7 @@ public JsonObject buildHttpsGranuleFile(JsonObject cnmFile){
         String uri = StringUtils.trim(cnmFile.get("uri").getAsString());
         AdapterLogger.LogInfo(this.className + " uri: " + uri);
         String path = uri.replace("https://", "");
+        path = path.replace("http://", ""); // Work-a-around to working with http links also
         //find the path by getting character from (first / plus 1) to lastIndex of /
         String url_path = path.substring(path.indexOf("/") + 1, path.lastIndexOf("/"));
 

diff --git a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java
@@ -147,4 +147,20 @@ public void testBuildSftpGranuleFile() throws Exception {
 
         assert(expectedJson.equals(outputJson.getAsJsonObject("output")));
     }
+
+    public void testBuildHttpGranuleFile() throws Exception {
+        ClassLoader classLoader = getClass().getClassLoader();
+        File inputJsonFile = new File(classLoader.getResource("http_input.json").getFile());
+        File expectedJsonFile = new File(classLoader.getResource("http_output.json").getFile());
+
+        String input = new String(Files.readAllBytes(inputJsonFile.toPath()));
+        String expected = new String(Files.readAllBytes(expectedJsonFile.toPath()));
+        JsonObject expectedJson = new JsonParser().parse(expected).getAsJsonObject();
+
+        CnmToGranuleHandler cnmToGranuleHandler = new CnmToGranuleHandler();
+        String output = cnmToGranuleHandler.PerformFunction(input, null);
+        JsonObject outputJson = new JsonParser().parse(output).getAsJsonObject();
+
+        assert(expectedJson.equals(outputJson.getAsJsonObject("output")));
+    }
 }
diff --git a/src/test/resources/http_input.json b/src/test/resources/http_input.json
@@ -0,0 +1,88 @@
+{
+  "input": {
+    "collection": "HLSL30",
+    "identifier": "2ffc5609-6ab4-4f58-b422-4d71ff43b269",
+    "duplicationid": "HLS.L30.T11VNK.2020259T185742.v2.0",
+    "version": "1.4",
+    "submissionTime": "2021-10-29T19:47:06Z",
+    "product": {
+      "name": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "dataVersion": "2.0",
+      "id": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "files": [
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "size": 6430,
+          "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "type": "metadata"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "size": 798505,
+          "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "size": 561155,
+          "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "size": 194484,
+          "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "type": "browse"
+        }
+      ]
+    }
+  },
+  "config": {
+    "collection": {
+      "files": [
+        {
+          "regex": ".*.h5$",
+          "sampleFileName": "HLSL30_product_0001-of-0050.h5",
+          "type": "data",
+          "bucket": "protected"
+        },
+        {
+          "regex": ".*.iso.xml$",
+          "sampleFileName": "HLSL30_product_0001-of-0019.iso.xml",
+          "type": "metadata",
+          "bucket": "protected"
+        },
+        {
+          "regex": ".*.cmr.json$",
+          "sampleFileName": "HLSL30_product_0001-of-0019.cmr.json",
+          "type": "metadata",
+          "bucket": "public"
+        }
+      ],
+      "name": "HLSL30",
+      "granuleIdExtraction": "^(.*)((\\.cmr\\.json)|(\\.h5)|(\\.h5\\.mp))$",
+      "granuleId": "^.*$",
+      "dataType": "HLSL30",
+      "provider_path": "HLSL30/",
+      "version": "2.0",
+      "updatedAt": 1552434051881,
+      "duplicateHandling": "replace",
+      "sampleFileName": "HLSL30_product_0001-of-0050.h5",
+      "createdAt": 1552434051881,
+      "meta": {
+        "required-files": [
+          ".*.h5$",
+          ".*.iso.xml$"
+        ]
+      }
+    }
+  }
+}
diff --git a/src/test/resources/http_output.json b/src/test/resources/http_output.json
@@ -0,0 +1,43 @@
+{
+  "granules": [
+    {
+      "granuleId": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "version": "2.0",
+      "dataType": "HLSL30",
+      "files": [
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 6430,
+          "checksumType": "SHA512",
+          "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3",
+          "type": "metadata"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 798505,
+          "checksumType": "SHA512",
+          "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 561155,
+          "checksumType": "SHA512",
+          "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 194484,
+          "checksumType": "SHA512",
+          "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3",
+          "type": "browse"
+        }
+      ]
+    }
+  ]
+}