Skip to content

Commit

Permalink
support RiverAvg and LakeAvg with geometry footprint and empty pass
Browse files Browse the repository at this point in the history
  • Loading branch information
Yen, David (398B-Affiliate) committed Aug 2, 2023
1 parent b8026e4 commit 5e0ee55
Show file tree
Hide file tree
Showing 26 changed files with 217 additions and 461 deletions.
5 changes: 3 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@ gradle -x test build
** -r --constructors-required-only : generate constructor for required field only
** -R remove old output
```aidl
jsonschema2pojo -s ./UMM-G1.6.3.json --target java-gen -p gov.nasa.cumulus.metadata.umm.model -a GSON -r -fdt true -R
jsonschema2pojo -s ./UMM-G1.6.3.json --target java-gen -p gov.nasa.cumulus.metadata.umm.model -a GSON -fdt
jsonschema2pojo -s ./ummg165.json --target java-gen -p gov.nasa.cumulus.metadata.umm.generated -a GSON -fdt -r -R
jsonschema2pojo -s ./ummg165.json --target java-gen -p gov.nasa.cumulus.metadata.umm.generated -a GSON -fdt
jsonshcema2pojo maven plugin is also configured within the pom.xml file
mvn compile // call plugin goal to generate pojo classes
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.440</version>
<version>1.12.515</version>
</dependency>
<!-- For AWS Secret Manager -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-secretsmanager</artifactId>
<version>1.12.440</version>
<version>1.12.515</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/CMRLambdaRestClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,8 @@ public HttpResponse validateUMMG(String provider, String granuleId, String strU
*/
public boolean isUMMGSpatialValid(String provider, String granuleId, String strUMMG)
throws URISyntaxException, IOException, ParseException {
AdapterLogger.LogInfo(this.className + " UMMG validation provider: "+ provider + " granuleId: " + granuleId +
" ummg: " + strUMMG);
HttpResponse httpResponse = validateUMMG(provider, granuleId, strUMMG);
int statusCode = httpResponse.getStatusLine().getStatusCode();
InputStream inputStream = httpResponse.getEntity().getContent();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/gov/nasa/cumulus/metadata/aggregator/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ public static class Metadata{
public static final String START_ORBIT = "startorbit";
public static final String END_ORBIT = "endorbit";
}
public static final String UMMG_VERSION = "1.6.3";
public static final String UMMG_VERSION = "1.6.5";
}
2 changes: 2 additions & 0 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/IsoMendsXPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public final class IsoMendsXPath extends IsoXPath {
// AscendingCrossing, StartLatitude, StartDirection, EndLatitude, EndDirection
public static final String ORBIT = "/gmi:MI_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicDescription[@id=\"Orbit\"]/gmd:geographicIdentifier/gmd:MD_Identifier/gmd:code/gco:CharacterString";

/** list of coordinates representing footprint */
public static final String LINE = "/gmi:MI_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_BoundingPolygon/gmd:polygon/gml:Polygon/gml:exterior/gml:LinearRing/gml:posList ";
public static final String GRANULE_INPUT = "/gmi:MI_Metadata/gmd:dataQualityInfo/gmd:DQ_DataQuality/gmd:lineage/gmd:LI_Lineage/gmd:source/gmi:LE_Source[gmd:description/gco:CharacterString[text()=\"GranuleInput\"]]/gmd:sourceCitation/gmd:CI_Citation/gmd:title/gmx:FileName";
public static final String CYCLE_PASS_TILE_SCENE = "/gmi:MI_Metadata/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicDescription[@id=\"SWOTTrack\"]/gmd:geographicIdentifier/gmd:MD_Identifier/gmd:code/gco:CharacterString";

Expand Down
6 changes: 2 additions & 4 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ public String PerformFunction(String input, Context context) throws Exception {
boolean isIsoFile = (iso != null);

mtfe = new MetadataFilesToEcho(isIsoFile);
//set the name/granuleId
mtfe.getGranule().setName(granuleId);
mtfe.setDatasetValues(collectionName, collectionVersion, rangeIs360, boundingBox, additionalAttributes);
if (granules != null && granules.size() > 0) {
mtfe.setGranuleFileSizeAndChecksum(granules);
Expand Down Expand Up @@ -182,10 +184,6 @@ public String PerformFunction(String input, Context context) throws Exception {
}
}

//set the name
mtfe.getGranule().setName(granuleId);


//write UMM-G to file
try {
mtfe.writeJson("/tmp/" + granuleId + ".cmr.json");
Expand Down
91 changes: 77 additions & 14 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataFilesToEcho.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import javax.xml.bind.DatatypeConverter;
import javax.xml.parsers.DocumentBuilder;
Expand Down Expand Up @@ -476,8 +477,25 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
if (qaPercentOutOfBoundsData != "" && BoundingTools.isParseable(qaPercentOutOfBoundsData)) {
((IsoGranule) granule).setQAPercentOutOfBoundsData(Double.parseDouble(qaPercentOutOfBoundsData));
}

((IsoGranule) granule).setOrbit(xpath.evaluate(IsoMendsXPath.ORBIT, doc));
/**
* first of all check if Orbit existed. If not, then
* extract the footprint/polygon from nc.iso.xml file and store the posList into the "line" Character
*/
String orbitStr = xpath.evaluate(IsoMendsXPath.ORBIT, doc);
if(!StringUtils.isEmpty(orbitStr)) {
((IsoGranule) granule).setOrbit(xpath.evaluate(IsoMendsXPath.ORBIT, doc));
} else {
try {
String line = xpath.evaluate(IsoMendsXPath.LINE, doc);
if (line != null && !line.isEmpty()) {
granule.getGranuleCharacterSet().add(createGranuleCharacter(line,"line"));
}
} catch (XPathExpressionException e) {
// Ignore if unable to parse for footprint since it isn't required for ingest
AdapterLogger.LogWarning(this.className + " Not able to extract MENDS footprint: " + e);
}
}
//extract and store Track Pass string
((IsoGranule) granule).setSwotTrack(xpath.evaluate(IsoMendsXPath.SWOT_TRACK, doc));

Source source = new Source();
Expand Down Expand Up @@ -535,8 +553,10 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
String mgrsId = xpath.evaluate(IsoMendsXPath.MGRS_ID, doc);
if (mgrsId != null && !mgrsId.equals("")) {
// If MGRS_ID field is not null, set as additional attribute
AdditionalAttributeType mgrsAttr = new AdditionalAttributeType("MGRS_TILE_ID", Collections.singletonList(mgrsId));

AdditionalAttributeType mgrsAttr = new AdditionalAttributeType();
mgrsAttr.setName("MGRS_TILE_ID");
mgrsAttr.setValues(Collections.singletonList(mgrsId));

List<AdditionalAttributeType> additionalAttributeTypes = ((IsoGranule) granule).getAdditionalAttributeTypes();
if (additionalAttributeTypes == null) {
additionalAttributeTypes = Collections.singletonList(mgrsAttr);
Expand All @@ -557,6 +577,44 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
return ((IsoGranule) granule);
}

public boolean isOrbitExisting(String orbitStr) {
if(StringUtils.isEmpty(StringUtils.trim(orbitStr))) {
return false;
} else{
try {
Pattern p = Pattern.compile("AscendingCrossing:\\s?(.*)\\s?StartLatitude:\\s?(.*)\\s?StartDirection:\\s?(.*)\\s?EndLatitude:\\s?(.*)\\s?EndDirection:\\s?(.*)");
Matcher m = p.matcher(orbitStr);
boolean foundOrbitalData = false;
foundOrbitalData = m.find();
String ascendingCrossingStr = StringUtils.trim(m.group(1));
String startLatitudeStr = StringUtils.trim(m.group(2));
String startDirectionStr = StringUtils.trim(m.group(3));
String endLatitudeStr = StringUtils.trim(m.group(4));
String endDirectionStr = StringUtils.trim(m.group(5));

/** to verify the Orbit string using as-tight-as-possible logic to make sure the orbitStr is parsable
* and not anyone of the item is "None"
*/
if (foundOrbitalData && BoundingTools.allParsable(ascendingCrossingStr, startLatitudeStr, endLatitudeStr) && (
!StringUtils.equalsIgnoreCase(ascendingCrossingStr, "None") &&
!StringUtils.equalsIgnoreCase(startLatitudeStr, "None") &&
!StringUtils.equalsIgnoreCase(startDirectionStr, "None") &&
!StringUtils.equalsIgnoreCase(endLatitudeStr, "None") &&
!StringUtils.equalsIgnoreCase(endDirectionStr, "None")
)) {
// only returning Orbit is found while the entire Orbit String check to be valid
return true;
} else {
return false;
}
} catch( java.lang.IllegalStateException | PatternSyntaxException ex) {
AdapterLogger.LogWarning(this.className + " error while checking if there is Orbit string: " + ex);
}
}
// again, only return true when a very tight pattern is valid
return false;
}

public List<AdditionalAttributeType> appendAdditionalAttributes(JSONObject metaAdditionalAttributes, NodeList additionalAttributesBlock){
/*
Scan through meta.additionalAttributes
Expand Down Expand Up @@ -668,13 +726,15 @@ public IsoGranule createIsoCyclePassTile(String cyclePassTileStr) {
AdapterLogger.LogError(this.className + " Creating TrackType with exception: " + UMMUtils.getStackTraceAsString(e));
throw e;
}

UmmgPojoFactory ummgPojoFactory = UmmgPojoFactory.getInstance();
additionalAttributeTypes=
ummgPojoFactory.trackTypeToAdditionalAttributeTypes(trackType);
}
// It is possible after all the above processing, cycle is present but passes is not (no pass in passes array)
// That is, we shall NOT create trackType at all. Otherwise, CMR will throw validation error
if (trackType.getCycle()!=null && trackType.getPasses()!=null && trackType.getPasses().size() >0) {
// from UMMG Schema 1.6.5, TrackType contains Cycle and Passes and ONLY Cycle is required
if (trackType.getCycle()!=null) {
((IsoGranule) granule).setTrackType(trackType);
((IsoGranule) granule).setAdditionalAttributeTypes(additionalAttributeTypes);
}
Expand Down Expand Up @@ -712,16 +772,18 @@ public TrackType createTrackType(String cyclePassTileStr, Pattern p_cycle) {
StringUtils.replace(passTilesStr,"[",""),"]","");
passTilesStr = passTilesStr.replaceAll("TILES\\s*:\\s*?", "");
String[] passTiles = StringUtils.split(passTilesStr, ",");
String passStr = StringUtils.trim(passTiles[0]);
trackPassTileType.setPass(NumberUtils.createInteger(UMMUtils.removeStrLeadingZeros(passStr)));
try {
List<String> tiles = getTiles(StringUtils.trim(passTiles[1]));
trackPassTileType.setTiles(tiles);
} catch (Exception e) {
AdapterLogger.LogWarning(this.className + " Continue processing after tile processing failed with " +
"exception: " + UMMUtils.getStackTraceAsString(e));
if(!StringUtils.isEmpty(passTilesStr) && passTiles.length >0) {
String passStr = StringUtils.trim(passTiles[0]);
trackPassTileType.setPass(NumberUtils.createInteger(UMMUtils.removeStrLeadingZeros(passStr)));
try {
List<String> tiles = getTiles(StringUtils.trim(passTiles[1]));
trackPassTileType.setTiles(tiles);
} catch (Exception e) {
AdapterLogger.LogWarning(this.className + " Continue processing after tile processing failed with " +
"exception: " + UMMUtils.getStackTraceAsString(e));
}
trackPassTileTypes.add(trackPassTileType);
}
trackPassTileTypes.add(trackPassTileType);
}
trackType.setPasses(trackPassTileTypes);
return trackType;
Expand Down Expand Up @@ -988,6 +1050,7 @@ public void readSentinelManifest(String file) throws ParserConfigurationExceptio
}
} catch (XPathExpressionException e) {
// Ignore if unable to parse for footprint since it isn't required for ingest
AdapterLogger.LogWarning(this.className + " Not able to extract footprint from SentinelManifest: " + e);
}

String cycle = StringUtils.trim(xpath.evaluate(ManifestXPath.CYCLE, doc));
Expand Down
40 changes: 27 additions & 13 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/UMMGranuleFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ public JSONObject defineGranule()
* Only when having gone through S6A Line to Polygon processing, then call UMMGPostProcessing.
*/
if(this.isLineFormattedPolygon) {
AdapterLogger.LogInfo(this.className + " Start post processing of UMMG by posting UMMG to CMR. If failed, put GBBox into UMMG");
granuleJson = UMMGPostProcessing(granuleJson);
}

Expand Down Expand Up @@ -396,6 +397,10 @@ private JSONObject exportTemporal() {
JSONObject range = new JSONObject();
range.put("BeginningDateTime", TimeConversion.convertDate(granule.getStartTime()).toString());
range.put("EndingDateTime", TimeConversion.convertDate(granule.getStopTime()).toString());
//TODO hard coded value for testing. Remove them later:
range.put("BeginningDateTime", "2022-12-16T22:41:48.323Z");
range.put("EndingDateTime", "2022-12-17T19:30:21.553Z");

temporal.put("RangeDateTime", range);
return temporal;
}
Expand Down Expand Up @@ -429,18 +434,26 @@ private JSONObject exportSpatial() throws ParseException{
addPolygon(geometry, polygon);
}
// Export Orbit
// Commented out for now since UMM v1.5 only allows for either Geometry or Orbit not both
JSONObject orbit = new JSONObject();
horizontalSpatialDomain.put("Orbit", orbit);
Pattern p = Pattern.compile("AscendingCrossing:\\s?(.*)\\s?StartLatitude:\\s?(.*)\\s?StartDirection:\\s?(.*)\\s?EndLatitude:\\s?(.*)\\s?EndDirection:\\s?(.*)");
Matcher m = p.matcher(((IsoGranule) granule).getOrbit());
foundOrbitalData = m.find();
if (foundOrbitalData && BoundingTools.allParsable(m.group(1), m.group(2), m.group(4))) {
orbit.put("AscendingCrossing", UMMUtils.longitudeTypeNormalizer(Double.parseDouble(m.group(1))));
orbit.put("StartLatitude", Double.parseDouble(m.group(2)));
orbit.put("StartDirection", m.group(3).trim());
orbit.put("EndLatitude", Double.parseDouble(m.group(4)));
orbit.put("EndDirection", m.group(5).trim());
// UMM v1.5 only allows for either Geometry or Orbit not both
/**
* Export Orbit
* UMM v1.5 only allows for either Geometry or Orbit not both. Only process orbit if the orbitString stored
* (during MetatdataFilesToEcho.readIsoxxxx()) is not empty or null
*/
String orbitStr = ((IsoGranule) granule).getOrbit();
if (!StringUtils.isEmpty(orbitStr)) {
JSONObject orbit = new JSONObject();
horizontalSpatialDomain.put("Orbit", orbit);
Pattern p = Pattern.compile("AscendingCrossing:\\s?(.*)\\s?StartLatitude:\\s?(.*)\\s?StartDirection:\\s?(.*)\\s?EndLatitude:\\s?(.*)\\s?EndDirection:\\s?(.*)");
Matcher m = p.matcher(orbitStr);
foundOrbitalData = m.find();
if (foundOrbitalData && BoundingTools.allParsable(m.group(1), m.group(2), m.group(4))) {
orbit.put("AscendingCrossing", UMMUtils.longitudeTypeNormalizer(Double.parseDouble(m.group(1))));
orbit.put("StartLatitude", Double.parseDouble(m.group(2)));
orbit.put("StartDirection", m.group(3).trim());
orbit.put("EndLatitude", Double.parseDouble(m.group(4)));
orbit.put("EndDirection", m.group(5).trim());
}
}

// Export track
Expand Down Expand Up @@ -586,10 +599,11 @@ private JSONObject exportSpatial() throws ParseException{
}

// Export footprint if it exists

Set<GranuleCharacter> granuleCharacters = granule.getGranuleCharacterSet();
for (GranuleCharacter granuleCharacter : granuleCharacters) {
if (granuleCharacter.getDatasetElement().getElementDD().getShortName().equals("line")) {
AdapterLogger.LogInfo(this.className + " Start processing line2Polygons : " + granuleCharacter.getValue() );
this.isLineFormattedPolygon = true;
geometry = line2Polygons(geometry,granuleCharacter.getValue());
break;
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/FootprintProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,8 +341,8 @@ public ExclusiveZoneType getExclusiveZones(Geometry geometry) {
boundary.setPoints(points);
boundaries.add(boundary);
}

ExclusiveZoneType exclusiveZoneType = new ExclusiveZoneType(boundaries);
ExclusiveZoneType exclusiveZoneType = new ExclusiveZoneType();
exclusiveZoneType.setBoundaries(boundaries);
return exclusiveZoneType;
}

Expand Down
20 changes: 2 additions & 18 deletions src/main/java/gov/nasa/cumulus/metadata/umm/generated/AdditionalAttributeType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -11,6 +12,7 @@
* A reference to an additional attribute in the parent collection. The attribute reference may contain a granule specific value that will override the value in the parent collection for this granule. An attribute with the same name must exist in the parent collection.
*
*/
@Generated("jsonschema2pojo")
public class AdditionalAttributeType {

/**
Expand All @@ -30,24 +32,6 @@ public class AdditionalAttributeType {
@Expose
private List<String> values = new ArrayList<String>();

/**
* No args constructor for use in serialization
*
*/
public AdditionalAttributeType() {
}

/**
*
* @param values
* @param name
*/
public AdditionalAttributeType(String name, List<String> values) {
super();
this.name = name;
this.values = values;
}

/**
* The additional attribute's name.
* (Required)
Expand Down
18 changes: 2 additions & 16 deletions src/main/java/gov/nasa/cumulus/metadata/umm/generated/BoundaryType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -11,6 +12,7 @@
* A boundary is set of points connected by straight lines representing a polygon on the earth. It takes a minimum of three points to make a boundary. Points must be specified in counter-clockwise order and closed (the first and last vertices are the same).
*
*/
@Generated("jsonschema2pojo")
public class BoundaryType {

/**
Expand All @@ -22,22 +24,6 @@ public class BoundaryType {
@Expose
private List<PointType> points = new ArrayList<PointType>();

/**
* No args constructor for use in serialization
*
*/
public BoundaryType() {
}

/**
*
* @param points
*/
public BoundaryType(List<PointType> points) {
super();
this.points = points;
}

/**
*
* (Required)
Expand Down
Loading

0 comments on commit 5e0ee55

Please sign in to comment.