From bfa7cef035d6e2e804d8cadec3192d91de47fc93 Mon Sep 17 00:00:00 2001 From: "Yen, David (398B-Affiliate)" Date: Tue, 14 May 2024 20:25:15 -0700 Subject: [PATCH] enhance code guard to only append relatedUrls when subTypeHashArray is not null or size zero. more unit test. --- .../aggregator/MetadataAggregatorLambda.java | 8 +- .../aggregator/factory/TaskConfigFactory.java | 8 +- .../processor/RelatedUrlsProcessor.java | 4 +- .../test/RelatedUrlsProcessorTest.java | 122 ++++++++++++ .../nasa/cumulus/metadata/test/UMMTest.java | 4 +- ...0251Z_20221026T184342Z_L8_30_v0.0.cmr.json | 173 ++++++++++++++++++ 6 files changed, 309 insertions(+), 10 deletions(-) create mode 100644 src/test/java/gov/nasa/cumulus/metadata/test/RelatedUrlsProcessorTest.java create mode 100644 src/test/resources/ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java index 46c7099..148b09e 100644 --- a/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java +++ b/src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java @@ -12,7 +12,6 @@ import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.HashSet; import java.util.Hashtable; import java.util.List; @@ -22,7 +21,6 @@ import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor; import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor; import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor; -import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum; import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum; import gov.nasa.cumulus.metadata.util.S3Utils; import org.apache.commons.io.FileUtils; @@ -196,8 +194,10 @@ public String PerformFunction(String input, Context context) throws Exception { String cmrFilePath = "/tmp/" + granuleId + ".cmr.json"; try { JSONObject granuleJson = mtfe.createJson(); - RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor(); - granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files); + if(taskConfigBO.getSubTypeHashArray().size() > 0) { + RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor(); + granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files); + } FileUtils.writeStringToFile(new File(cmrFilePath), granuleJson.toJSONString(), StandardCharsets.UTF_8); } catch (IOException e) { AdapterLogger.LogError(this.className + " mtfe.writeJson error:" + e.getMessage()); diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java index 6a1887e..d08bf0d 100644 --- a/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java +++ b/src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java @@ -18,10 +18,14 @@ public static TaskConfigBO createTaskConfigBO(JSONObject config) { TaskConfigBO taskConfigBO = new TaskConfigBO(); //Construct isoXMLSpatialTypeHashset JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType"); - taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray)); + if(isoXMLSpatialTypeJsonArray !=null && isoXMLSpatialTypeJsonArray.size()>0) { + taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray)); + } //Construct subTypeHashArray JSONArray subTypeTypeJsonArray = (JSONArray) config.get("relatedUrlSubTypeMap"); - taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray)); + if(subTypeTypeJsonArray!=null && subTypeTypeJsonArray.size()>0) { + taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray)); + } return taskConfigBO; } diff --git a/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java b/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java index 9a4b63a..4134927 100644 --- a/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java +++ b/src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java @@ -3,7 +3,6 @@ import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO; import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; import java.util.ArrayList; import java.util.HashMap; @@ -21,6 +20,9 @@ public void RelatedUrlsProcessor() { public JSONObject appendSubTypes(JSONObject granuleJson, TaskConfigBO taskConfigBO, JSONArray files) { ArrayList> subTypeHashArray=taskConfigBO.getSubTypeHashArray(); + if(subTypeHashArray==null || subTypeHashArray.size()==0) { + return granuleJson; + } JSONArray relateUrlsArray = (JSONArray) granuleJson.get("RelatedUrls"); // 1: Loop through the existing RelatedUrls array to find any match for the subType regex. // for the finding match, set Subtype to diff --git a/src/test/java/gov/nasa/cumulus/metadata/test/RelatedUrlsProcessorTest.java b/src/test/java/gov/nasa/cumulus/metadata/test/RelatedUrlsProcessorTest.java new file mode 100644 index 0000000..e71be29 --- /dev/null +++ b/src/test/java/gov/nasa/cumulus/metadata/test/RelatedUrlsProcessorTest.java @@ -0,0 +1,122 @@ +package gov.nasa.cumulus.metadata.test; + +import gov.nasa.cumulus.metadata.aggregator.IsoGranule; +import gov.nasa.cumulus.metadata.aggregator.MetadataAggregatorLambda; +import gov.nasa.cumulus.metadata.aggregator.MetadataFilesToEcho; +import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO; +import gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory; +import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor; +import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.junit.Before; +import org.junit.Test; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class RelatedUrlsProcessorTest { + TaskConfigBO taskConfigBO = new TaskConfigBO(); + + @Before + public void setUp() throws Exception { + // Setup ArrayList> subTypeHashArray which is a data structure presenting relatedUrlSubTypeMap + // Read relatedUrlSubTypeMap from collection_config/OPERA_L3_DSWX-S1_PROVISIONAL_V0.json which is a collection configuration json + ClassLoader classLoader = getClass().getClassLoader(); + File file = new File(classLoader.getResource("collection_config/OPERA_L3_DSWX-S1_PROVISIONAL_V0.json").getFile()); + JSONParser parser = new JSONParser(); + Object obj = parser.parse(new FileReader(file.getAbsolutePath())); + JSONObject collectionJSONObject = (JSONObject) obj; + // extract the relatedUrlSubTypeMap from collection configuration meta structure + JSONArray subTypeHasyArrayJson = (JSONArray)(((JSONObject)(collectionJSONObject.get("meta"))).get("relatedUrlSubTypeMap")); + JSONObject config = new JSONObject(); + config.put("relatedUrlSubTypeMap", subTypeHasyArrayJson); + taskConfigBO = TaskConfigFactory.createTaskConfigBO(config); + } + + @Test + public void testAppendSubTypes() throws IOException, ParseException, ParserConfigurationException, + SAXException, XPathExpressionException, URISyntaxException { + ClassLoader classLoader = getClass().getClassLoader(); + File file = new File(classLoader.getResource("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.iso.xml").getFile()); + File cfgFile = new File(classLoader.getResource("OPERA_L3_DSWX-HLS_PROVISIONAL_V0_test_1.cmr.cfg").getFile()); + MetadataFilesToEcho mfte = new MetadataFilesToEcho(true); + mfte.getGranule().setName("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0"); + Document doc = null; + XPath xpath = null; + mfte.readConfiguration(cfgFile.getAbsolutePath()); + doc = mfte.makeDoc(file.getAbsolutePath()); + xpath = mfte.makeXpath(doc); + IsoGranule isoGranule = mfte.readIsoMendsMetadataFile("s3://mybucket/mygranule.nc", doc, xpath); + // use a unrelated .mp file to patch required field so mfte.createJson() would work. + File file2 = new File(classLoader.getResource("JA1_GPN_2PeP374_172_20120303_112035_20120303_121638.nc.mp").getFile()); + try { + mfte.readCommonMetadataFile(file2.getAbsolutePath(), "s3://a/path/to/s3"); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + mfte.getGranule().setName("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0"); + JSONObject granuleJson = mfte.createJson(); + + org.json.simple.JSONArray files = new JSONArray(); + JSONObject f1 = new JSONObject(); + JSONObject f2 = new JSONObject(); + JSONObject f3 = new JSONObject(); + // construct the files:[] array under payload + f1.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif"); + f1.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif"); + f1.put("type", "data"); + f1.put("checksumType", "md5"); + f2.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif"); + f2.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif"); + f2.put("type", "data"); + f2.put("checksumType", "md5"); + f3.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B03_CONF.tif"); + f3.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B03_CONF.tif"); + f3.put("type", "data"); + f3.put("checksumType", "md5"); + files.add(f1);files.add(f2);files.add(f3); + RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor(); + // Use appendSubTypes to generate the new Json including certain file items with subType + JSONObject newGranuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson,taskConfigBO, files); + JSONArray relatedUrls =(JSONArray) newGranuleJson.get("RelatedUrls"); + ArrayList> subTypeHashArray = taskConfigBO.getSubTypeHashArray(); + /** + * We loop through the RelatedUrls array. for each relatedUrl item. If the URL field match the regex of + * a subTypeHash (which for example contains "regex": "^.*_B01_WTR\\.tif$", + * "subType": "BROWSE IMAGE SOURCE"), the Subtype field should match the + * subTypeHash's subType value + */ + for(int i = 0; i < relatedUrls.size(); i++) + { + JSONObject relatedUrl = (JSONObject)relatedUrls.get(i); + String URLStr=relatedUrl.get("URL").toString(); + for(HashMap subTypeHash : subTypeHashArray) { + if(URLStr.matches(subTypeHash.get("regex"))) { + assertEquals(relatedUrl.get("Subtype"), subTypeHash.get("subType")); + } + } + } + // Compare the newGranuleJson with the pre-saved cmr.json file + assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json", newGranuleJson)); + } +} diff --git a/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java b/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java index 30d0f52..808264c 100644 --- a/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java +++ b/src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java @@ -852,8 +852,6 @@ public void testStringMatch() { String regex = "^.*_B02_BWTR\\.tif$"; String s3File = "s3://src/test/resources/tester/asdaffaf_B02_BWTR.tif"; boolean matched = s3File.matches(regex); - System.out.println("matched? " + matched); + assertEquals(matched, true); } - - } diff --git a/src/test/resources/ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json b/src/test/resources/ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json new file mode 100644 index 0000000..0008427 --- /dev/null +++ b/src/test/resources/ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json @@ -0,0 +1,173 @@ +{ + "InputGranules": [ + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B02.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B03.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B04.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B05.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B06.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B07.tif", + "/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.Fmask.tif", + "/home/conda/input_dir/dem.vrt", + "/home/conda/input_dir/dem_0.tif", + "/home/conda/input_dir/landcover.tif", + "/home/conda/input_dir/worldcover.vrt", + "/home/conda/input_dir/worldcover_0.tif" + ], + "TemporalExtent": { + "RangeDateTime": { + "EndingDateTime": "2012-03-03T12:16:38.727Z", + "BeginningDateTime": "2012-03-03T11:20:35.236Z" + } + }, + "MetadataSpecification": { + "Version": "1.6.6", + "URL": "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6", + "Name": "UMM-G" + }, + "GranuleUR": "OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0", + "ProviderDates": [ + { + "Type": "Insert", + "Date": "2024-05-15T03:19:06.144Z" + }, + { + "Type": "Update", + "Date": "2024-05-15T03:19:06.154Z" + } + ], + "SpatialExtent": { + "HorizontalSpatialDomain": { + "Geometry": { + "BoundingRectangles": [ + { + "WestBoundingCoordinate": 0.238, + "SouthBoundingCoordinate": -66.147, + "EastBoundingCoordinate": 359.846, + "NorthBoundingCoordinate": 66.142 + } + ] + }, + "Track": { + "Cycle": 374, + "Passes": [ + { + "Pass": 172 + } + ] + } + } + }, + "AdditionalAttributes": [ + { + "Values": [ + "HLS.L30.T14RNV.2021249T170251.v2.0" + ], + "Name": "HlsDataset" + }, + { + "Values": [ + "LC08_L1TP_027038_20210906_20210915_02_T1; LC08_L1TP_027039_20210906_20210915_02_T1" + ], + "Name": "SensorProductID" + }, + { + "Values": [ + "Lasrc; Lasrc" + ], + "Name": "Accode" + }, + { + "Values": [ + "135.857670779298" + ], + "Name": "MeanSunAzimuthAngle" + }, + { + "Values": [ + "32.3676212035439" + ], + "Name": "MeanSunZenithAngle" + }, + { + "Values": [ + "30.8893421483768" + ], + "Name": "NBAR_SolarZenith" + }, + { + "Values": [ + "106.186652634788" + ], + "Name": "MeanViewAzimuthAngle" + }, + { + "Values": [ + "5.12004424472291" + ], + "Name": "MeanViewZenithAngle" + }, + { + "Values": [ + "67" + ], + "Name": "SpatialCoverage" + }, + { + "Values": [ + "76" + ], + "Name": "PercentCloudCover" + }, + { + "Values": [ + "T14RNV" + ], + "Name": "MGRS_TILE_ID" + } + ], + "PGEVersionClass": { + "PGEName": "DSWX_HLS_PGE", + "PGEVersion": "1.0.0-rc.5" + }, + "DataGranule": { + "DayNightFlag": "Unspecified", + "Identifiers": [ + { + "Identifier": "OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0", + "IdentifierType": "ProducerGranuleId" + }, + { + "Identifier": "0.5", + "IdentifierType": "Other", + "IdentifierName": "SASVersionId" + }, + { + "Identifier": "1.0.0-rc.5", + "IdentifierType": "Other", + "IdentifierName": "PGEVersionId" + } + ], + "ProductionDateTime": "2016-03-16T03:02:03.000Z" + }, + "CollectionReference": { + "Version": "0.0", + "ShortName": "OPERA_L3_DSWX-HLS_PROVISIONAL_V0" + }, + "RelatedUrls": [ + { + "Type": "GET DATA", + "Description": "The base directory location for the granule.", + "URL": "s3://a/path/to/s3" + }, + { + "Type": "GET DATA", + "Subtype": "BROWSE IMAGE SOURCE", + "URL": "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif" + }, + { + "Type": "GET DATA", + "Subtype": "IceBridge Portal", + "URL": "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif" + } + ] +} \ No newline at end of file