Merge pull request #75 from podaac/release/8.6.0

Release/8.6.0
podaac · May 20, 2024 · b5fc731 · b5fc731
2 parents 4c8f1e0 + ed41164
commit b5fc731
Show file tree

Hide file tree

Showing 35 changed files with 133 additions and 142 deletions.
diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: '8.0.232'
+          java-version: '11.0.6'
       - uses: gradle/gradle-build-action@v1
         with:
           gradle-version: 8.0.1

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.6.0]
+### Added
+- **PODAAC-5876**
+  - Update to use CMA 2.0.0, thus allowing 2.0.3 layer for lambda
+  - Update build to use java 11
+  - To generate java 11 compatible UMMG schema POJOs, jsonschema2pojo shall make use of command line parameter: --target-version 1.11 
+### Deprecated
+### Removed
+### Fixed
+- **PODAAC-5857**
+  - Fixed the issue so footprint and bbx do not always appear together while isoXmlSpatial is configured to footprint only
+  - remove request.close() statement in CMRLambdaRestClient so the response could be pulled out and logged properly instead of always logging apache chunk read error due to the http channel was closed.
+  - adding more parameters to Jsonschema2pojo plugin to generate proper pojo from ummg schema
+### Security
+- Snyk: Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.544 to 1.12.641
+
 ## [8.5.0]
 ### Added
 - **PODAAC-5594**

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-8.5.0
+8.6.0
diff --git a/build.gradle b/build.gradle
@@ -1,6 +1,6 @@
 apply plugin: 'java'
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
+sourceCompatibility = 11
+targetCompatibility = 11
 
 dependencies {
     implementation fileTree(dir: 'target/dependency/', include: '*.jar')

diff --git a/pom.xml b/pom.xml
@@ -14,16 +14,16 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>gov.nasa.podaac</groupId>
   <artifactId>cumulus-metadata-aggregator</artifactId>
-  <version>8.5.0</version>
+  <version>8.6.0-rc.7</version>
   <packaging>jar</packaging>
 
   <name>Metadata-Aggregator</name>
   <description>This module contains the main software for the Metadata-Aggregator lambda.</description>
   <url>TODO</url>
 
   <properties>
-    <maven.compiler.source>1.8</maven.compiler.source>
-    <maven.compiler.target>1.8</maven.compiler.target>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
   </properties>
@@ -39,12 +39,12 @@
     <dependency>
       <groupId>gov.nasa.earthdata</groupId>
       <artifactId>cumulus-message-adapter</artifactId>
-      <version>1.3.9</version>
+      <version>2.0.0</version>
     </dependency>
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
-      <version>1.12.544</version>
+      <version>1.12.641</version>
     </dependency>
     <!-- For AWS Secret Manager -->
     <dependency>
@@ -133,6 +133,12 @@
       <version>4.5.1</version>
       <scope>test</scope>
     </dependency>
+    <!-- https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api -->
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <version>2.3.1</version>
+    </dependency>
   </dependencies>
   <build>
     <sourceDirectory>src/main/java</sourceDirectory>
@@ -172,14 +178,16 @@
       <plugin>
         <groupId>org.jsonschema2pojo</groupId>
         <artifactId>jsonschema2pojo-maven-plugin</artifactId>
-        <version>1.0.2</version>
+        <version>1.2.1</version>
         <configuration>
           <sourceDirectory>${basedir}/src/main/resources/jsonschema</sourceDirectory>
           <outputDirectory>src/main/java</outputDirectory>
           <targetPackage>gov.nasa.cumulus.metadata.umm.generated</targetPackage>
           <sourceType>jsonschema</sourceType>
           <annotationStyle>GSON</annotationStyle>
-          <includeConstructors>true</includeConstructors>
+          <includeConstructors>false</includeConstructors>
+          <targetVersion>11</targetVersion>
+          <includeGeneratedAnnotation>false</includeGeneratedAnnotation>
           <formatDateTimes>true</formatDateTimes>
           <formatDates>true</formatDates>
           <formatDateTimes>true</formatDateTimes>

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRLambdaRestClient.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRLambdaRestClient.java
@@ -76,7 +76,6 @@ public CMRLambdaRestClient(String cert, String pass, String tknHost,
         this.echoHost = cmrHost;
         URIBuilder uriBuilder = new URIBuilder(tknHost);
         this.tokenHost = uriBuilder.setPath(uriBuilder.getPath() + "/gettoken").build().normalize().toString();
-        AdapterLogger.LogInfo(this.className + " final token url:" + this.tokenHost);
         this.validHost = this.tokenHost.replaceAll("gettoken", "validate");
         this.region = region;
         this.tknBucket = tknBucket;
@@ -105,15 +104,13 @@ public String getToken()
         JSONObject jsonTkn = readToken();
         long fileTime = getTokenStartTime(jsonTkn);
         if (jsonTkn == null || sessionExpired(fileTime, runTime)) {
-            AdapterLogger.LogDebug(this.className + " Generating new NAMS token...");
             try {
                 this.token = buildToken(runTime);
             } catch (Exception e) {
                 //ERROR getting new token from NAMS.
                 throw new IOException("Could not retrieve token..." + e);
             }
         } else {
-            AdapterLogger.LogDebug(this.className + " Session active, using saved token");
             this.token = (String) jsonTkn.get("token");
         }
         return this.token;
@@ -261,7 +258,6 @@ private JSONObject readToken()
         }
         String localTokenFilePath = s3Utils.download(this.region, this.tknBucket, this.tknFilePath,
                 Paths.get(this.workingDir, "token.json").toString());
-        AdapterLogger.LogInfo(this.className + " downloaded token file to local:" + localTokenFilePath);
         JSONObject json = null;
         try (FileReader reader = new FileReader(String.valueOf(localTokenFilePath))) {
             json = (JSONObject) parser.parse(reader);
@@ -278,7 +274,6 @@ private void writeToken(long timeStamp, String token)
         JSONObject json = new JSONObject();
         json.put("authTime", timeStamp);
         json.put("token", token);
-        AdapterLogger.LogDebug(this.className + " Writing new token to file");
         Path localTknFilePath = Files.write(Paths.get(this.workingDir, "token.json"),
                 json.toString().getBytes());
         s3Utils.upload(region, this.tknBucket, this.tknFilePath, new File(localTknFilePath.toString()) );
@@ -302,7 +297,6 @@ private HttpResponse send(String url, HttpEntity entity)
         request.setHeader("Content-Type", content_type);
         // Send the request
         HttpResponse response = httpClient.execute(request);
-        request.releaseConnection();
         return response;
     }
 
@@ -314,7 +308,6 @@ public HttpResponse validateUMMG(String provider, String granuleId,  String strU
             String validateUMMGUri = uriBuilder.setPath(uriBuilder.getPath() + "/ingest/providers/"
                     + provider +"/validate/granule/" + granuleId)
                     .build().normalize().toString();
-            AdapterLogger.LogDebug("validateUri:" + validateUMMGUri);
             HttpEntity httpEntity = new StringEntity(strUMMG, "utf-8");
             HttpResponse httpResponse = send(validateUMMGUri, httpEntity);
             return httpResponse;

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRRestClient.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRRestClient.java
@@ -191,8 +191,6 @@ private void send(String url, HttpEntity entity) throws IOException {
         } else {
             logHttpResponse(response, HttpOp.SEND);
         }
-        // close the connection
-        request.releaseConnection();
     }
 
     public void closeScrollSession(String scrollJson) throws IOException {

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/IsoGranule.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/IsoGranule.java
@@ -20,7 +20,7 @@ public class IsoGranule extends UMMGranule {
     private List<String> inputGranules;
     private String PGEVersionClass;
 
-    private IsoType isoType;
+
 
     public IsoGranule() {
         this.identifiers = new HashMap<>();
@@ -130,12 +130,4 @@ public String getPGEVersionClass() {
     public void setPGEVersionClass(String PGEVersionClass) {
         this.PGEVersionClass = PGEVersionClass;
     }
-
-    public IsoType getIsoType() {
-        return isoType;
-    }
-
-    public void setIsoType(IsoType isoType) {
-        this.isoType = isoType;
-    }
 }
diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java
@@ -10,7 +10,6 @@
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
 import javax.xml.bind.DatatypeConverter;
 import javax.xml.parsers.DocumentBuilder;
@@ -324,12 +323,12 @@ public void readIsoMetadataFile(String file, String s3Location) throws ParserCon
         // if we get here, we have the bare minimum fields already populated,
         // so try and parse the rest of the granule metadata...
         try {
+			((IsoGranule) this.granule).setIsoType(isoType);
             if (isoType == IsoType.MENDS) {
                 AdapterLogger.LogInfo("Found MENDS file");
                 readIsoMendsMetadataFile(s3Location, doc, xpath);
             } else if (isoType == IsoType.SMAP) {
                 AdapterLogger.LogInfo("Found SMAP file");
-				((IsoGranule) this.granule).setIsoType(isoType);
                 readIsoSmapMetadataFile(s3Location, doc, xpath);
             } else {
                 AdapterLogger.LogWarning(isoType.name() + " didn't match any expected ISO type, skipping optional " +
@@ -486,7 +485,6 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
 
 		((IsoGranule) granule).setOrbit(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.ORBIT, "IsoMendsXPath.ORBIT"));
 		((IsoGranule) granule).setSwotTrack(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.SWOT_TRACK, "IsoMendsXPath.SWOT_TRACK"));
-
 		Source source = new Source();
 		source.setSourceShortName(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.PLATFORM, "IsoMendsXPath.PLATFORM"));
 
@@ -511,7 +509,7 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
 		String  cyclePassTileSceneStr =StringUtils.trim(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.CYCLE_PASS_TILE_SCENE, "IsoMendsXPath.CYCLE_PASS_TILE_SCENE"));
 		if(!StringUtils.isBlank(cyclePassTileSceneStr)) {
 			try {
-				createIsoCyclePassTile(cyclePassTileSceneStr);
+				granule = createIsoCyclePassTile(cyclePassTileSceneStr);
 			} catch (Exception e) {
 				// Since TrackType which contains Cycle Pass Tile and Scenes is not a required field
 				// we catch exception with printStackTrace to know the exact line throwing error

diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/UMMGranuleFile.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/UMMGranuleFile.java
@@ -420,7 +420,8 @@ private JSONObject exportTemporal() {
      */
     private boolean shouldAddBBx(Granule granule) {
         boolean shouldAddBBx = false;
-        if(granule !=null && granule instanceof  gov.nasa.cumulus.metadata.aggregator.UMMGranule) {
+
+        if(granule !=null && granule.getIsoType() == null) {
             shouldAddBBx = true;
         }
         // if the granule object is IsoGranule type and it is SMAP mission, then we check if polygon was added.
@@ -430,6 +431,7 @@ private boolean shouldAddBBx(Granule granule) {
         &&  StringUtils.isNotEmpty(((IsoGranule) granule).getPolygon())) {
                 shouldAddBBx = false;
         }
+        AdapterLogger.LogInfo(this.className + " shouldAddBBx:" + shouldAddBBx);
         return shouldAddBBx;
     }
 
@@ -453,6 +455,7 @@ private JSONObject exportSpatial() throws ParseException{
                 AdapterLogger.LogInfo(this.className + " nc.iso.xml footprint processing ... ");
                 this.isLineFormattedPolygon = true;
                 geometry = line2Polygons(geometry,polygon);
+                horizontalSpatialDomain.put("Geometry", geometry);
             }
             if(this.isoXMLSpatialTypeEnumHashSet.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)) {
                 AdapterLogger.LogDebug(this.className + "UMMGranuleFile.exportSpatial ORBIT Processing");
@@ -478,13 +481,18 @@ private JSONObject exportSpatial() throws ParseException{
                 isoBBoxAlreadyProcessed = true;
                 horizontalSpatialDomain = this.appendBoundingRectangles(geometry, horizontalSpatialDomain);
             }
-            // Export track
-            if (((IsoGranule) granule).getSwotTrack() != "") {
+            // Export track for isoXML SMAP
+            // look into : IsoSmapXPath.SWOT_TRACK and how we use it to grab and store coordinate string to granule's
+            // swotTrack string.  We also store swotTrack to MENDs granule and IsoXPath.SWOT_TRACK and IsoMendsXPath.CYCLE_PASS_TILE_SCENE
+            // these 2 values are being used to "extrac" cycle pass string from eith MENDs or SMAP. However, smap cycle and pass
+            // are being processed below
+            if (((IsoGranule) granule).getSwotTrack() != "" && granule.getIsoType() == IsoType.SMAP) {
                 JSONObject track = new JSONObject();
                 horizontalSpatialDomain.put("Track", track);
                 Pattern trackPattern = Pattern.compile("Cycle:\\s(.*)\\sPass:\\s(.*)\\sTile:\\s(.*)");
                 Matcher trackMatcher = trackPattern.matcher(((IsoGranule) granule).getSwotTrack());
                 if (trackMatcher.find()) {
+                    AdapterLogger.LogDebug("SWOT track found cycle");
                     track.put("Cycle", Integer.parseInt(trackMatcher.group(1)));
                     JSONArray passes = new JSONArray();
                     track.put("Passes", passes);
@@ -510,17 +518,15 @@ private JSONObject exportSpatial() throws ParseException{
             }
         } // end of processing IsoGranule
 
+        // following is a large block code to deal with None IsoXML (MENDs or SMAP) bounding box
         // We can only include orbital or bounding-box data, not both
         // if iso Bounding Box already processed in logic above, then don't enter this block
         if (foundOrbitalData == false && !isoBBoxAlreadyProcessed) {
-
             horizontalSpatialDomain.put("Geometry", geometry);
-
             JSONArray boundingRectangles = new JSONArray();
-            geometry.put("BoundingRectangles", boundingRectangles);
 
             double north = 0, south = 0, east = 0, west = 0;
-            if(granule !=null && granule instanceof  gov.nasa.cumulus.metadata.aggregator.UMMGranule) {
+            if(granule !=null &&  granule.getIsoType() == null) {
                 east = ((UMMGranule) granule).getBbxEasternLongitude() != null ?
                         ((UMMGranule) granule).getBbxEasternLongitude() : 0;
                 west = ((UMMGranule) granule).getBbxWesternLongitude() != null?
@@ -565,7 +571,6 @@ private JSONObject exportSpatial() throws ParseException{
                 // and make sure we turn off the rangeIs360 flag
                 this.rangeIs360 = false;
             }
-
             BigDecimal nrth = new BigDecimal(north);
             BigDecimal sth = new BigDecimal(south);
             nrth = nrth.setScale(3, RoundingMode.HALF_UP);
@@ -609,9 +614,17 @@ private JSONObject exportSpatial() throws ParseException{
                     }
                 }
             }
-        }
 
-        // Export track if cycle and pass exists
+            if (boundingRectangles.size() > 0) {
+                geometry.put("BoundingRectangles", boundingRectangles);
+            }
+        }
+        // Export TrackType
+        // MENDS  : ISO
+        // Or Non-iso track : such as MODIS_A and Sentinel-6
+        // for instance, iso MENDS and Sentinel-6 and MODIS_A are having TrackType created within MetadataFilesToEcho class
+        // if cycle and pass exists. MENDs is taking adavangate of UMMG generated pojo : TrackType
+        // which is stored within the UMMGranule
         if (granule instanceof UMMGranule) {
             /**
              * Track include cycle and passes(array).

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/AdditionalAttributeType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/AdditionalAttributeType.java
@@ -3,7 +3,6 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import javax.annotation.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -12,7 +11,6 @@
  * A reference to an additional attribute in the parent collection. The attribute reference may contain a granule specific value that will override the value in the parent collection for this granule. An attribute with the  same name must exist in the parent collection.
  * 
  */
-@Generated("jsonschema2pojo")
 public class AdditionalAttributeType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundaryType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundaryType.java
@@ -3,7 +3,6 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import javax.annotation.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -12,7 +11,6 @@
  * A boundary is set of points connected by straight lines representing a polygon on the earth. It takes a minimum of three points to make a boundary. Points must be specified in counter-clockwise order and closed (the first and last vertices are the same).
  * 
  */
-@Generated("jsonschema2pojo")
 public class BoundaryType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundingRectangleType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundingRectangleType.java
@@ -1,7 +1,6 @@
 
 package gov.nasa.cumulus.metadata.umm.generated;
 
-import javax.annotation.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -10,7 +9,6 @@
  * This entity holds the horizontal spatial coverage of a bounding box.
  * 
  */
-@Generated("jsonschema2pojo")
 public class BoundingRectangleType {
 
     /**

diff --git a/src/main/java/gov/nasa/cumulus/metadata/umm/generated/ExclusiveZoneType.java b/src/main/java/gov/nasa/cumulus/metadata/umm/generated/ExclusiveZoneType.java
@@ -3,7 +3,6 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import javax.annotation.Generated;
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -12,7 +11,6 @@
  * Contains the excluded boundaries from the GPolygon.
  * 
  */
-@Generated("jsonschema2pojo")
 public class ExclusiveZoneType {
 
     /**