Feature/PODAAC-5770: selective footprint, orbit, bbox procesing from …

…iso.xml (#65)

* support RiverAvg and LakeAvg with geometry footprint and empty pass

* commented out the hard coded granule beginning/ending datetime

* added RiverAvg sample iso.xml file for unit testing

* ummtest using schema version 1.6.5

* CHANGELOG

* support BasinID

* process nc.iso.xml based on collection config : "isoXmlSpatial": ["footprint"], // can be "footprint", "bbox", or "orbit"

* CHANGELOG

* move PODAAC-5770 to the Fixed section of CHANGELOG

* iso.xml use the POLYGON path, not the LINE

* code modification based on comments

* fixed compiling error

* New utility to perform XPath extraction with better logging

* better handle XPath extraction logging and remove MENDSIsoXMLSpatialTypeConstant

---------

Co-authored-by: Yen, David (398B-Affiliate) <[email protected]>

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- **PODAAC-5594**
+  - Support BasinID
+- **PODAAC-5770**
+  - use meta.isoXMLSpatialType to configure the collection should process the combination of footprint, orbit and bbox
+- **PODAAC-5717**
+  - Upgrade to UMMG 1.6.5
+  - support empty Pass in Cycle/Pass/Tile string
+### Deprecated
+### Removed
+### Fixed
+### Security
+
+- 
 ## [8.4.0]
 ### Added
 - Update metadata aggregator to add description to image variables from image processor and test tig forge processor

README.md

@@ -54,8 +54,9 @@ gradle -x test build
     ** -r  --constructors-required-only : generate constructor for required field only
     ** -R   remove old output 
  ```aidl
-  jsonschema2pojo -s ./UMM-G1.6.3.json --target java-gen -p gov.nasa.cumulus.metadata.umm.model  -a GSON -r -fdt true -R
-  jsonschema2pojo -s ./UMM-G1.6.3.json --target java-gen -p gov.nasa.cumulus.metadata.umm.model  -a GSON -fdt 
+  
+  jsonschema2pojo -s ./ummg165.json --target java-gen -p gov.nasa.cumulus.metadata.umm.generated -a GSON -fdt -r -R
+  jsonschema2pojo -s ./ummg165.json --target java-gen -p gov.nasa.cumulus.metadata.umm.generated  -a GSON -fdt 
   
   jsonshcema2pojo maven plugin is also configured within the pom.xml file
   mvn compile    // call plugin goal to generate pojo classes

pom.xml

@@ -44,13 +44,13 @@
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
-      <version>1.12.440</version>
+      <version>1.12.544</version>
     </dependency>
     <!-- For AWS Secret Manager -->
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-secretsmanager</artifactId>
-      <version>1.12.440</version>
+      <version>1.12.544</version>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>

src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRLambdaRestClient.java

@@ -337,6 +337,8 @@ public HttpResponse validateUMMG(String provider, String granuleId,  String strU
      */
     public boolean isUMMGSpatialValid(String provider, String granuleId, String strUMMG)
             throws URISyntaxException, IOException, ParseException {
+        AdapterLogger.LogInfo(this.className + " UMMG validation provider: "+ provider + " granuleId: " + granuleId +
+        " ummg: " + strUMMG);
         HttpResponse httpResponse = validateUMMG(provider, granuleId, strUMMG);
         int statusCode = httpResponse.getStatusLine().getStatusCode();
         InputStream inputStream = httpResponse.getEntity().getContent();

src/main/java/gov/nasa/cumulus/metadata/aggregator/Constants.java

@@ -6,5 +6,5 @@ public static class Metadata{
         public static final String START_ORBIT = "startorbit";
         public static final String END_ORBIT = "endorbit";
     }
-    public static final String UMMG_VERSION = "1.6.3";
+    public static final String UMMG_VERSION = "1.6.5";
 }

src/main/java/gov/nasa/cumulus/metadata/aggregator/IsoMendsXPath.java

@@ -34,6 +34,7 @@ public final class IsoMendsXPath extends IsoXPath {
     // AscendingCrossing, StartLatitude, StartDirection, EndLatitude, EndDirection
     public static final String ORBIT = "/gmi:MI_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicDescription[@id=\"Orbit\"]/gmd:geographicIdentifier/gmd:MD_Identifier/gmd:code/gco:CharacterString";
 
+    /** list of coordinates representing footprint */
     public static final String GRANULE_INPUT = "/gmi:MI_Metadata/gmd:dataQualityInfo/gmd:DQ_DataQuality/gmd:lineage/gmd:LI_Lineage/gmd:source/gmi:LE_Source[gmd:description/gco:CharacterString[text()=\"GranuleInput\"]]/gmd:sourceCitation/gmd:CI_Citation/gmd:title/gmx:FileName";
     public static final String CYCLE_PASS_TILE_SCENE = "/gmi:MI_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicDescription[@id=\"SWOTTrack\"]/gmd:geographicIdentifier/gmd:MD_Identifier/gmd:code/gco:CharacterString";
 

src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java

@@ -11,12 +11,14 @@
 import java.security.UnrecoverableKeyException;
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.List;
 
 import gov.nasa.cumulus.metadata.aggregator.processor.DMRPPProcessor;
 import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor;
 import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor;
+import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
 import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum;
 import gov.nasa.cumulus.metadata.util.S3Utils;
 import org.apache.commons.io.FileUtils;
@@ -55,6 +57,10 @@ public String PerformFunction(String input, Context context) throws Exception {
 		 * this will help the logic in postIngestProcess function.
 		 */
 		this.setWorkFlowType((String) config.get("stateMachine"));
+		// This is a switch to determine, shall footprint, orbit or boundingbox shall be processed from iso.xml
+		// while ingesting swot collections
+		JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
+		HashSet isoXMLSpatialTypeHashSet = createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray);
 
 
 		String isoRegex = (String) config.get("isoRegex");
@@ -149,7 +155,9 @@ public String PerformFunction(String input, Context context) throws Exception {
 		MetadataFilesToEcho mtfe;
         boolean isIsoFile = (iso != null);
 
-        mtfe = new MetadataFilesToEcho(isIsoFile);
+        mtfe = new MetadataFilesToEcho(isIsoFile, isoXMLSpatialTypeHashSet);
+		//set the name/granuleId
+		mtfe.getGranule().setName(granuleId);
         mtfe.setDatasetValues(collectionName, collectionVersion, rangeIs360, boundingBox, additionalAttributes);
         if (granules != null && granules.size() > 0) {
             mtfe.setGranuleFileSizeAndChecksum(granules);
@@ -182,10 +190,6 @@ public String PerformFunction(String input, Context context) throws Exception {
 			}
 		}
 
-		//set the name
-		mtfe.getGranule().setName(granuleId);
-
-
 		//write UMM-G to file
 		try {
 			mtfe.writeJson("/tmp/" + granuleId + ".cmr.json");
@@ -234,6 +238,33 @@ public String PerformFunction(String input, Context context) throws Exception {
 		return returnable.toJSONString();
 	}
 
+	public HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
+		HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
+		// if not containing isoXMLTypes, then return an empty HashSet
+		if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
+			return isoSpatialTypes;
+		}
+		isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
+			String t = (String) item;
+			MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
+			isoSpatialTypes.add(en);
+		});
+		AdapterLogger.LogDebug(this.className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
+		return isoSpatialTypes;
+	}
+
+	public String getIsoXMLSpatialTypeStr(String token) {
+		final String trimmedToken = StringUtils.trim(token);
+		String s;
+		try {
+			s = MENDsIsoXMLSpatialTypeEnum.getEnumValuList().stream()
+					.filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
+		} catch (java.util.NoSuchElementException e) {
+			s = "";
+		}
+		return s;
+	}
+
 	/**
 	 * get S3 fileStaging direction from S3 full key
 	 *