Skip to content

Commit

Permalink
process nc.iso.xml based on collection config : "isoXmlSpatial": ["fo…
Browse files Browse the repository at this point in the history 
…otprint"], // can be "footprint", "bbox", or "orbit"
  • Loading branch information
Yen, David (398B-Affiliate) committed Sep 7, 2023
1 parent 562eda1 commit c7ce833
Show file tree
Hide file tree
Showing 7 changed files with 261 additions and 66 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.515</version>
<version>1.12.544</version>
</dependency>
<!-- For AWS Secret Manager -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-secretsmanager</artifactId>
<version>1.12.515</version>
<version>1.12.544</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
Expand Down
36 changes: 35 additions & 1 deletion src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,15 @@
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;

import gov.nasa.cumulus.metadata.aggregator.constant.MENDsIsoXMLSpatialTypeConstant;
import gov.nasa.cumulus.metadata.aggregator.processor.DMRPPProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum;
import gov.nasa.cumulus.metadata.util.S3Utils;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -55,6 +58,10 @@ public String PerformFunction(String input, Context context) throws Exception {
* this will help the logic in postIngestProcess function.
*/
this.setWorkFlowType((String) config.get("stateMachine"));
// This is a switch to determine, shall footprint, orbit or boundingbox shall be processed from iso.xml
// while ingesting swot collections
JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
HashSet isoXMLSpatialTypeHashSet = createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray);


String isoRegex = (String) config.get("isoRegex");
Expand Down Expand Up @@ -149,7 +156,7 @@ public String PerformFunction(String input, Context context) throws Exception {
MetadataFilesToEcho mtfe;
boolean isIsoFile = (iso != null);

mtfe = new MetadataFilesToEcho(isIsoFile);
mtfe = new MetadataFilesToEcho(isIsoFile, isoXMLSpatialTypeHashSet);
//set the name/granuleId
mtfe.getGranule().setName(granuleId);
mtfe.setDatasetValues(collectionName, collectionVersion, rangeIs360, boundingBox, additionalAttributes);
Expand Down Expand Up @@ -232,6 +239,33 @@ public String PerformFunction(String input, Context context) throws Exception {
return returnable.toJSONString();
}

public HashSet<MENDsIsoXMLSpatialTypeEnum> createIsoXMLSpatialTypeSet(JSONArray isoXMLSpatialTypeConfigJSONArray) throws IllegalArgumentException{
HashSet<MENDsIsoXMLSpatialTypeEnum> isoSpatialTypes = new HashSet<>();
// if not containing isoXMLTypes, then return an empty HashSet
if(isoXMLSpatialTypeConfigJSONArray == null || isoXMLSpatialTypeConfigJSONArray.size()==0) {
return isoSpatialTypes;
}
isoXMLSpatialTypeConfigJSONArray.forEach(item -> {
String t = (String) item;
MENDsIsoXMLSpatialTypeEnum en = MENDsIsoXMLSpatialTypeEnum.getEnum(getIsoXMLSpatialTypeStr(t));
isoSpatialTypes.add(en);
});
AdapterLogger.LogDebug(this.className + " isoSpatialTypes HashSet: " + isoSpatialTypes);
return isoSpatialTypes;
}

public String getIsoXMLSpatialTypeStr(String token) {
final String trimmedToken = StringUtils.trim(token);
String s;
try {
s = MENDsIsoXMLSpatialTypeConstant.isoXMLSpatialTypeList.stream()
.filter(e -> StringUtils.equals(trimmedToken, e)).findFirst().get();
} catch (java.util.NoSuchElementException e) {
s = "";
}
return s;
}

/**
* get S3 fileStaging direction from S3 full key
*
Expand Down
91 changes: 55 additions & 36 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import javax.xml.xpath.*;

import gov.nasa.cumulus.metadata.aggregator.factory.UmmgPojoFactory;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import gov.nasa.cumulus.metadata.umm.generated.AdditionalAttributeType;
import gov.nasa.cumulus.metadata.umm.generated.TrackPassTileType;
import gov.nasa.cumulus.metadata.umm.generated.TrackType;
Expand Down Expand Up @@ -63,6 +64,7 @@ public class MetadataFilesToEcho {
boolean isIsoFile = false;
JSONObject additionalAttributes = null;
UmmgPojoFactory ummgPojoFactory = UmmgPojoFactory.getInstance();
HashSet<MENDsIsoXMLSpatialTypeEnum> isoXMLSpatialTypeEnumHashSet = new HashSet<>();

public MetadataFilesToEcho() {
this(false);
Expand All @@ -76,6 +78,15 @@ public MetadataFilesToEcho(boolean isIso) {
this.granule = new UMMGranule();
}

public MetadataFilesToEcho(boolean isIso, HashSet inputIsoXMLSpatialTypeHashSet) {
this.isIsoFile = isIso;
if (isIsoFile)
this.granule = new IsoGranule();
else
this.granule = new UMMGranule();
this.isoXMLSpatialTypeEnumHashSet = inputIsoXMLSpatialTypeHashSet;
}

//this method reads the configuration file (per dataset) sent to this class (.cfg)
public void readConfiguration(String file) throws IOException, ParseException {
JSONParser parser = new JSONParser();
Expand Down Expand Up @@ -261,14 +272,9 @@ public void readCommonMetadataFile(String file, String s3Location) throws IOExce
public void setGranuleFileSizeAndChecksum(JSONArray input_granules) {

JSONArray files = (JSONArray)((JSONObject)input_granules.get(0)).get("files");

AdapterLogger.LogDebug(this.className + " setGranuleFileSizeAndChecksum files[]:" + files);
for(Object f: files){
JSONObject file = (JSONObject)f;
AdapterLogger.LogDebug(this.className + " UMM-G GranuleArchive filename:" + (String)file.get("fileName"));
AdapterLogger.LogDebug(this.className + " UMM-G GranuleArchive filesize:" + ((Double) file.get("size")).longValue());
AdapterLogger.LogDebug(this.className + " UMM-G GranuleArchive checksum:" + (String)file.get("checksum"));
AdapterLogger.LogDebug(this.className + " UMM-G GranuleArchive checksumType:" + (String)file.get("checksumType"));
AdapterLogger.LogDebug(this.className + " UMM-G GranuleArchive type:" + (String)file.get("type"));
UMMGranuleArchive uga = new UMMGranuleArchive();
uga.setName((String)file.get("fileName"));
uga.setFileSize(((Double) file.get("size")).longValue());
Expand Down Expand Up @@ -412,17 +418,10 @@ private void parseRequiredFields(Document doc, XPath xpath, IsoType iso) throws
}
}

public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPath xpath) throws XPathExpressionException {

if (xpath.evaluate(IsoMendsXPath.NORTH_BOUNDING_COORDINATE, doc) != "") {
setGranuleBoundingBox(
Double.parseDouble(xpath.evaluate(IsoMendsXPath.NORTH_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.SOUTH_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.EAST_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.WEST_BOUNDING_COORDINATE, doc)));
}
((IsoGranule) granule).setPolygon(xpath.evaluate(IsoMendsXPath.POLYGON, doc));

public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPath xpath)
throws XPathExpressionException {
/* read footprint, bounding box , orbit based on cumulus task_config field */
this.granule = readFootprintOrbitBBox(doc, xpath);
NodeList nodes = (NodeList) xpath.evaluate(IsoMendsXPath.DATA_FILE, doc, XPathConstants.NODESET);
for (int i = 0; i < nodes.getLength(); i++) {
Element dataFile = (Element) nodes.item(i);
Expand Down Expand Up @@ -477,24 +476,7 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
if (qaPercentOutOfBoundsData != "" && BoundingTools.isParseable(qaPercentOutOfBoundsData)) {
((IsoGranule) granule).setQAPercentOutOfBoundsData(Double.parseDouble(qaPercentOutOfBoundsData));
}
/**
* first of all check if Orbit existed. If not, then
* extract the footprint/polygon from nc.iso.xml file and store the posList into the "line" Character
*/
String orbitStr = xpath.evaluate(IsoMendsXPath.ORBIT, doc);
if(!StringUtils.isEmpty(orbitStr)) {
((IsoGranule) granule).setOrbit(xpath.evaluate(IsoMendsXPath.ORBIT, doc));
} else {
try {
String line = xpath.evaluate(IsoMendsXPath.LINE, doc);
if (line != null && !line.isEmpty()) {
granule.getGranuleCharacterSet().add(createGranuleCharacter(line,"line"));
}
} catch (XPathExpressionException e) {
// Ignore if unable to parse for footprint since it isn't required for ingest
AdapterLogger.LogWarning(this.className + " Not able to extract MENDS footprint: " + e);
}
}

//extract and store Track Pass string
((IsoGranule) granule).setSwotTrack(xpath.evaluate(IsoMendsXPath.SWOT_TRACK, doc));

Expand Down Expand Up @@ -577,6 +559,43 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
return ((IsoGranule) granule);
}

public IsoGranule readFootprintOrbitBBox( Document doc, XPath xpath) throws XPathExpressionException{
if(this.isoXMLSpatialTypeEnumHashSet.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)) {
AdapterLogger.LogDebug(this.className + " Based on MENDsIsoXMLSpatialTypeEnum, processing BBOX");
if (xpath.evaluate(IsoMendsXPath.NORTH_BOUNDING_COORDINATE, doc) != "") {
setGranuleBoundingBox(
Double.parseDouble(xpath.evaluate(IsoMendsXPath.NORTH_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.SOUTH_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.EAST_BOUNDING_COORDINATE, doc)),
Double.parseDouble(xpath.evaluate(IsoMendsXPath.WEST_BOUNDING_COORDINATE, doc)));
}
}
/**
* There shall be no more logic of "if there is orbit, then there shall be no footprint" anymore
* this function will solely depends on cumulus collection config to determine the collection is
* GEODETIC, CARTICIAN , ORBIT
* if isoXMLSpatialTypeEnumHashSet contains FOOTPRINT, means, the collection is either GEODETIC or CARTICIAN
*/
if(this.isoXMLSpatialTypeEnumHashSet.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)) {
AdapterLogger.LogDebug(this.className + " Based on MENDsIsoXMLSpatialTypeEnum, processing FOOTPRINT");
try {
((IsoGranule) granule).setPolygon(xpath.evaluate(IsoMendsXPath.POLYGON, doc));
String line = xpath.evaluate(IsoMendsXPath.LINE, doc);
if (line != null && !line.isEmpty()) {
granule.getGranuleCharacterSet().add(createGranuleCharacter(line,"line"));
}
} catch (XPathExpressionException e) {
// Ignore if unable to parse for footprint since it isn't required for ingest
AdapterLogger.LogWarning(this.className + " Not able to extract MENDS footprint: " + e);
}
}
if(this.isoXMLSpatialTypeEnumHashSet.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)) {
AdapterLogger.LogDebug(this.className + " Based on MENDsIsoXMLSpatialTypeEnum, processing ORBIT");
((IsoGranule) granule).setOrbit(xpath.evaluate(IsoMendsXPath.ORBIT, doc));
}
return ((IsoGranule) granule);
}

public boolean isOrbitExisting(String orbitStr) {
if(StringUtils.isEmpty(StringUtils.trim(orbitStr))) {
return false;
Expand Down Expand Up @@ -1105,7 +1124,7 @@ private void setGranuleBoundingBox(double north, double south, double east, doub
public JSONObject createJson()
throws ParseException, IOException, URISyntaxException {
granule.setIngestTime(new Date());
UMMGranuleFile granuleFile = new UMMGranuleFile(granule, dataset, rangeIs360);
UMMGranuleFile granuleFile = new UMMGranuleFile(granule, dataset, rangeIs360, this.isoXMLSpatialTypeEnumHashSet);
JSONObject granuleJson = granuleFile.defineGranule();
return granuleJson;
}
Expand Down
Loading

0 comments on commit c7ce833

Please sign in to comment.