From 0488fb98936e191f596fbe55519a8d847bd8dfb6 Mon Sep 17 00:00:00 2001 From: "Yen, David (398B-Affiliate)" Date: Fri, 10 May 2024 17:14:24 -0700 Subject: [PATCH] CHANGELOG, upgrade AWS java libraries. Fixed unit tests --- CHANGELOG.md | 17 ++ pom.xml | 4 +- .../test/MetadataAggregatorLambdaTest.java | 23 +- .../test/MetadataFilesToEchoTest.java | 205 ------------------ .../nasa/cumulus/metadata/test/UMMTest.java | 35 +-- 5 files changed, 48 insertions(+), 236 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e99e185..fd7ab33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- **PODAAC-6181** + - add relatedUrl subType BROWSE IMAGE SOURCE + - Update build to use java 11 + - To generate java 11 compatible UMMG schema POJOs, jsonschema2pojo shall make use of command line parameter: --target-version 1.11 +### Deprecated +### Removed +### Fixed +- **PODAAC-5857** + - Fixed the issue so footprint and bbx do not always appear together while isoXmlSpatial is configured to footprint only + - remove request.close() statement in CMRLambdaRestClient so the response could be pulled out and logged properly instead of always logging apache chunk read error due to the http channel was closed. + - adding more parameters to Jsonschema2pojo plugin to generate proper pojo from ummg schema +### Security +- Snyk: Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.544 to 1.12.641 + + ## [8.6.0] ### Added - **PODAAC-5876** diff --git a/pom.xml b/pom.xml index f4b7f9a..952ddd5 100755 --- a/pom.xml +++ b/pom.xml @@ -44,13 +44,13 @@ com.amazonaws aws-java-sdk-s3 - 1.12.701 + 1.12.719 com.amazonaws aws-java-sdk-secretsmanager - 1.12.701 + 1.12.719 org.apache.httpcomponents diff --git a/src/test/java/gov/nasa/cumulus/metadata/test/MetadataAggregatorLambdaTest.java b/src/test/java/gov/nasa/cumulus/metadata/test/MetadataAggregatorLambdaTest.java index 6648be2..0b2d78a 100644 --- a/src/test/java/gov/nasa/cumulus/metadata/test/MetadataAggregatorLambdaTest.java +++ b/src/test/java/gov/nasa/cumulus/metadata/test/MetadataAggregatorLambdaTest.java @@ -1,8 +1,11 @@ package gov.nasa.cumulus.metadata.test; import gov.nasa.cumulus.metadata.aggregator.MetadataAggregatorLambda; +import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO; +import gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory; import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum; import org.json.simple.JSONArray; +import org.json.simple.JSONObject; import org.json.simple.parser.ParseException; import org.junit.Before; import org.junit.Test; @@ -41,23 +44,17 @@ public void testGetConceptId() throws ParseException { assertEquals(conceptId, "G1238611022-POCUMULUS"); } - @Test - public void testGetIsoXMLSpatialTypeStr() { - MetadataAggregatorLambda lambda = new MetadataAggregatorLambda(); - assertEquals(lambda.getIsoXMLSpatialTypeStr("footprint"), "footprint"); - assertEquals(lambda.getIsoXMLSpatialTypeStr("orbit"), "orbit"); - assertEquals(lambda.getIsoXMLSpatialTypeStr("bbox"), "bbox"); - assertEquals(lambda.getIsoXMLSpatialTypeStr("xxxx"), ""); - } - @Test public void testCreateIsoXMLSpatialTypeSet() { MetadataAggregatorLambda lambda = new MetadataAggregatorLambda(); org.json.simple.JSONArray array = new JSONArray(); array.add("footprint"); array.add("orbit"); + JSONObject config = new JSONObject(); + config.put("isoXMLSpatialType", array); + TaskConfigBO taskConfigBO = TaskConfigFactory.createTaskConfigBO(config); //HashSet h = lambda.createIsoXMLSpatialTypeSet("[footprint,orbit]"); - HashSet h = lambda.createIsoXMLSpatialTypeSet(array); + HashSet h = taskConfigBO.getIsoXMLSpatialTypeHashSet(); assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); @@ -69,8 +66,10 @@ public void testCreateIsoXMLSpatialTypeSet() { array.add("bbox"); array.add("eebb"); array.add("ccmm"); - //h = lambda.createIsoXMLSpatialTypeSet("[footprint,orbit,bbox,eebb,ccmm]"); - h = lambda.createIsoXMLSpatialTypeSet(array); + config = new JSONObject(); + config.put("isoXMLSpatialType", array); + taskConfigBO = TaskConfigFactory.createTaskConfigBO(config); + h = taskConfigBO.getIsoXMLSpatialTypeHashSet(); assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); diff --git a/src/test/java/gov/nasa/cumulus/metadata/test/MetadataFilesToEchoTest.java b/src/test/java/gov/nasa/cumulus/metadata/test/MetadataFilesToEchoTest.java index 77673c4..2905e62 100644 --- a/src/test/java/gov/nasa/cumulus/metadata/test/MetadataFilesToEchoTest.java +++ b/src/test/java/gov/nasa/cumulus/metadata/test/MetadataFilesToEchoTest.java @@ -994,209 +994,4 @@ public void testCreateJsonSwotCalVal() throws IOException, ParseException, XPath // load pre-saved file and perform json comparison assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/swotCalVal/SWOTCalVal_WM_ADCP_L0_RiverRay1_20220727T191701_20220727T192858_20220920T142800_swotCalVal_ummg.json", granule)); } - @Test - public void testSWOTCreateJsonSWOTIsoXMLSpatialType() throws IOException, ParseException, XPathExpressionException, ParserConfigurationException, SAXException, URISyntaxException { - // Create isoXmlSpatial Hashtable which contains footprint, orbit then passed the hashtable to MetadataFilesToEcho contructor - MetadataAggregatorLambda lambda = new MetadataAggregatorLambda(); - org.json.simple.JSONArray array = new JSONArray(); - array.add("footprint"); - array.add("orbit"); - array.add("bbox"); - - HashSet h = lambda.createIsoXMLSpatialTypeSet(array); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.NONE)); - - ClassLoader classLoader = getClass().getClassLoader(); - File file = new File(classLoader.getResource("swotIsoXMLSpatialTypeTestData/SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01.nc.iso.xml").getFile()); - - // constructor to set isIso =true and the isoXmlSpatialHashtable - MetadataFilesToEcho mfte = new MetadataFilesToEcho(true, h); - mfte.getGranule().setName("SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01"); - File cfgFile = new File(classLoader.getResource("MODIS_T-JPL-L2P-v2014.0.cmr.cfg").getFile()); - mfte.readConfiguration(cfgFile.getAbsolutePath()); - mfte.readIsoMetadataFile(file.getAbsolutePath(), "s3://fake_bucket/fake_dir/fake.nc.iso.xml"); - - JSONObject granule = mfte.createJson(); - JSONArray polygonPoints =((JSONArray)((JSONObject) ((JSONObject)((JSONArray)((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("GPolygons")).get(0)).get("Boundary")).get("Points")); - Double latitude=(Double)((JSONObject)polygonPoints.get(0)).get("Latitude"); - Double longitude=(Double)((JSONObject)polygonPoints.get(0)).get("Longitude"); - assert latitude.doubleValue() ==-77.089598228; - assert longitude.doubleValue() == -121.56652283899999; - - latitude=(Double)((JSONObject)polygonPoints.get(27)).get("Latitude"); - longitude=(Double)((JSONObject)polygonPoints.get(27)).get("Longitude"); - assert latitude.doubleValue()==56.734470077; - assert longitude.doubleValue()==-21.668558564000023; - - latitude=(Double)((JSONObject)polygonPoints.get(64)).get("Latitude"); - longitude=(Double)((JSONObject)polygonPoints.get(64)).get("Longitude"); - assert latitude.doubleValue()==-77.089598228; - assert longitude.doubleValue()== -121.56652283899999; - - JSONObject orbit = ((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Orbit")); - assert ((Double)orbit.get("StartLatitude"))==-77.66; //-77.66 - assert ((Double)orbit.get("EndLatitude"))==77.66;// -> {Double@5567} 77.66 - assert ((Double)orbit.get("AscendingCrossing"))==-38.05; //-> {Double@5569} -38.05 - assertTrue(StringUtils.equals((String)orbit.get("StartDirection"), "A")); //-> A - assertTrue(StringUtils.equals((String)orbit.get("EndDirection"), "A")); //EndDirection -> A - - JSONObject bbox = (JSONObject)((JSONArray)((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("BoundingRectangles")).get(0); - assert ((BigDecimal)bbox.get("WestBoundingCoordinate")).compareTo(new BigDecimal("-121.76947499999999990905052982270717620849609375")) ==0; - assert ((BigDecimal)bbox.get("SouthBoundingCoordinate")).compareTo(new BigDecimal("-78.271941999999995687176124192774295806884765625"))==0; - assert ((BigDecimal)bbox.get("EastBoundingCoordinate")).compareTo(new BigDecimal("45.675058000000035463017411530017852783203125"))==0; - assert ((BigDecimal)bbox.get("NorthBoundingCoordinate")).compareTo(new BigDecimal("78.272067999999990206561051309108734130859375"))==0; - - - // verify the pre-saved ummg.json file is equal to the granule json just built - assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/swotIsoXMLSpatialType/SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01_footprintOrbitBBox.json", granule)); - /** - * Test isoXMLSpatial:[footprint] - */ - clearVariables4IsoXMLSpatialTest(array, h, granule); // clear variables first - array.add("footprint"); - h = lambda.createIsoXMLSpatialTypeSet(array); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.NONE)); - mfte = new MetadataFilesToEcho(true, h); - mfte.getGranule().setName("SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01"); - cfgFile = new File(classLoader.getResource("MODIS_T-JPL-L2P-v2014.0.cmr.cfg").getFile()); - mfte.readConfiguration(cfgFile.getAbsolutePath()); - mfte.readIsoMetadataFile(file.getAbsolutePath(), "s3://fake_bucket/fake_dir/fake.nc.iso.xml"); - granule = mfte.createJson(); - polygonPoints =((JSONArray)((JSONObject) ((JSONObject)((JSONArray)((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("GPolygons")).get(0)).get("Boundary")).get("Points")); - latitude=(Double)((JSONObject)polygonPoints.get(0)).get("Latitude"); - longitude=(Double)((JSONObject)polygonPoints.get(0)).get("Longitude"); - assert latitude.doubleValue() ==-77.089598228; - assert longitude.doubleValue() == -121.56652283899999; - - latitude=(Double)((JSONObject)polygonPoints.get(27)).get("Latitude"); - longitude=(Double)((JSONObject)polygonPoints.get(27)).get("Longitude"); - assert latitude.doubleValue()==56.734470077; - assert longitude.doubleValue()==-21.668558564000023; - - latitude=(Double)((JSONObject)polygonPoints.get(64)).get("Latitude"); - longitude=(Double)((JSONObject)polygonPoints.get(64)).get("Longitude"); - assert latitude.doubleValue()==-77.089598228; - assert longitude.doubleValue()== -121.56652283899999; - - orbit = ((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Orbit")); - assert orbit==null; - - bbox = (JSONObject) (((JSONObject)((JSONObject)((JSONObject) granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("BoundingRectangles")); - assert bbox == null; - // load pre-saved file and perform json comparison - assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/swotIsoXMLSpatialType/SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01_footprint.json", granule)); - /** - * Test isoXMLSpatial:[bbox] - */ - clearVariables4IsoXMLSpatialTest(array, h, granule); // clear variables first - array.add("bbox"); - h = lambda.createIsoXMLSpatialTypeSet(array); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.NONE)); - mfte = new MetadataFilesToEcho(true, h); - mfte.getGranule().setName("SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01"); - cfgFile = new File(classLoader.getResource("MODIS_T-JPL-L2P-v2014.0.cmr.cfg").getFile()); - mfte.readConfiguration(cfgFile.getAbsolutePath()); - mfte.readIsoMetadataFile(file.getAbsolutePath(), "s3://fake_bucket/fake_dir/fake.nc.iso.xml"); - granule = mfte.createJson(); - JSONArray GPolygons = (JSONArray)((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("GPolygons"); - assert GPolygons ==null; - - orbit = ((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Orbit")); - assert orbit==null; - - bbox = (JSONObject)((JSONArray)((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")) - .get("BoundingRectangles")).get(0); - assert ((BigDecimal)bbox.get("WestBoundingCoordinate")).compareTo(new BigDecimal("-121.76947499999999990905052982270717620849609375")) ==0; - assert ((BigDecimal)bbox.get("SouthBoundingCoordinate")).compareTo(new BigDecimal("-78.271941999999995687176124192774295806884765625"))==0; - assert ((BigDecimal)bbox.get("EastBoundingCoordinate")).compareTo(new BigDecimal("45.675058000000035463017411530017852783203125"))==0; - assert ((BigDecimal)bbox.get("NorthBoundingCoordinate")).compareTo(new BigDecimal("78.272067999999990206561051309108734130859375"))==0; - // load pre-saved file and perform json comparison - assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/swotIsoXMLSpatialType/SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01_bbox.json", granule)); - /** - * Test isoXMLSpatial:[orbit] - */ - clearVariables4IsoXMLSpatialTest(array, h, granule); // clear variables first - array.add("orbit"); - h = lambda.createIsoXMLSpatialTypeSet(array); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.FOOTPRINT)); - assertTrue(h.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.BBOX)); - assertFalse(h.contains(MENDsIsoXMLSpatialTypeEnum.NONE)); - mfte = new MetadataFilesToEcho(true, h); - mfte.getGranule().setName("SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01"); - cfgFile = new File(classLoader.getResource("MODIS_T-JPL-L2P-v2014.0.cmr.cfg").getFile()); - mfte.readConfiguration(cfgFile.getAbsolutePath()); - mfte.readIsoMetadataFile(file.getAbsolutePath(), "s3://fake_bucket/fake_dir/fake.nc.iso.xml"); - granule = mfte.createJson(); - JSONObject geometry = ((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry")); - assert geometry ==null; - - orbit = ((JSONObject)((JSONObject)((JSONObject)granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Orbit")); - assert ((Double)orbit.get("StartLatitude"))==-77.66; //-77.66 - assert ((Double)orbit.get("EndLatitude"))==77.66;// -> {Double@5567} 77.66 - assert ((Double)orbit.get("AscendingCrossing"))==-38.05; //-> {Double@5569} -38.05 - assertTrue(StringUtils.equals((String)orbit.get("StartDirection"), "A")); //-> A - assertTrue(StringUtils.equals((String)orbit.get("EndDirection"), "A")); //EndDirection -> A - // No Geometry object since there is neither footprint nor bbox - bbox = (((JSONObject)((JSONObject)((JSONObject) granule - .get("SpatialExtent")) - .get("HorizontalSpatialDomain")) - .get("Geometry"))); - assert bbox == null; - // load pre-saved file and perform json comparison - assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/swotIsoXMLSpatialType/SWOT_L2_LR_SSH_Basic_006_143_20231107T150730_20231107T155607_PIB0_01_orbit.json", granule)); - } - - private void clearVariables4IsoXMLSpatialTest(JSONArray isoXMLSpatialArray, HashSet isoXMLSpatialHashSet, JSONObject granule) { - isoXMLSpatialArray.clear(); - isoXMLSpatialHashSet.clear(); - granule.clear(); - } } \ No newline at end of file diff --git a/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java b/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java index 59d5d2f..3b16525 100644 --- a/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java +++ b/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java @@ -6,6 +6,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.math.BigDecimal; import java.net.URISyntaxException; import java.text.DecimalFormat; import java.util.ArrayList; @@ -530,10 +531,10 @@ public void testSentinelManifestL0TooFewCoordinates() //Geometry/BoundingRectangles JSONObject gbbx = (JSONObject)boundingCoordinates.get(0); - assertEquals(gbbx.get("WestBoundingCoordinate"), Double.valueOf(-180)); - assertEquals(gbbx.get("SouthBoundingCoordinate"), Double.valueOf(-90.00)); - assertEquals(gbbx.get("EastBoundingCoordinate"), Double.valueOf(180.00)); - assertEquals(gbbx.get("NorthBoundingCoordinate"), Double.valueOf(90.00)); + assertEquals(gbbx.get("WestBoundingCoordinate"), BigDecimal.valueOf(-180.0)); + assertEquals(gbbx.get("SouthBoundingCoordinate"), BigDecimal.valueOf(-90.00)); + assertEquals(gbbx.get("EastBoundingCoordinate"), BigDecimal.valueOf(180.00)); + assertEquals(gbbx.get("NorthBoundingCoordinate"), BigDecimal.valueOf(90.00)); } @Test @@ -595,11 +596,11 @@ public void testSentinelManifestIDLErrorCase() Object firstBoundBoxObj = ((JSONArray)geom.get("BoundingRectangles")).get(0); JSONObject bouningdbox = (JSONObject)firstBoundBoxObj; - assertEquals(Double.valueOf(-180.0), ((Double) bouningdbox.get("WestBoundingCoordinate"))); - assertEquals(Double.valueOf(-90.0), ((Double) bouningdbox.get("SouthBoundingCoordinate"))); + assertEquals(BigDecimal.valueOf(-180.0), ((BigDecimal) bouningdbox.get("WestBoundingCoordinate"))); + assertEquals(BigDecimal.valueOf(-90.0), ((BigDecimal) bouningdbox.get("SouthBoundingCoordinate"))); - assertEquals(Double.valueOf(180.0), ((Double) bouningdbox.get("EastBoundingCoordinate"))); - assertEquals(Double.valueOf(90.0), ((Double) bouningdbox.get("NorthBoundingCoordinate"))); + assertEquals(BigDecimal.valueOf(180.0), ((BigDecimal) bouningdbox.get("EastBoundingCoordinate"))); + assertEquals(BigDecimal.valueOf(90.0), ((BigDecimal) bouningdbox.get("NorthBoundingCoordinate"))); } @Test @@ -842,15 +843,15 @@ private JSONObject parseXfduManifest(String testFile, String testConfigFile, Str mtfe.readSentinelManifest(testFilePath); mtfe.getGranule().setName(granuleId); -// JSONObject granuleJson = mtfe.createJson(); -// return granuleJson; - //write UMM-G to file - mtfe.writeJson(testDir + "/" + granuleId + ".cmr.json"); - - //the CMR file should have the following values... - JSONParser parser = new JSONParser(); - Object obj = parser.parse(new FileReader(testDir + "/" + granuleId + ".cmr.json")); - return (JSONObject) obj; + JSONObject granuleJson = mtfe.createJson(); + return granuleJson; +// //write UMM-G to file +// mtfe.writeJson(testDir + "/" + granuleId + ".cmr.json"); +// +// //the CMR file should have the following values... +// JSONParser parser = new JSONParser(); +// Object obj = parser.parse(new FileReader(testDir + "/" + granuleId + ".cmr.json")); +// return (JSONObject) obj; } @Test