From 11581a5384fe1be8643ffa1b64691f68f749c0be Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 3 Jun 2022 19:14:17 -0500 Subject: [PATCH 1/3] Add group path to series index mapping in root attributes Fixes #126. The root level attributes (if they are written) will now contain a "groups" dictionary that maps each group with multiscales metadata to the corresponding Bio-Formats series index (== Image index in METADATA.ome.xml). --- .../bioformats2raw/Converter.java | 34 ++++++++++--- .../bioformats2raw/test/ZarrTest.java | 51 +++++++++++++++++++ 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 53f05ca..63b5b97 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -665,6 +665,32 @@ public void convert() scaleFormatString = "%s/%s/%d/%d"; } + // fileset level metadata + if (!noRootGroup) { + final ZarrGroup root = ZarrGroup.create(getRootPath()); + Map attributes = new HashMap(); + attributes.put("bioformats2raw.layout", LAYOUT); + + // record the path to each series (multiscales) and the corresponding + // series (OME-XML Image) index + // using the index as the key would mean that the index is stored + // as a string instead of an integer + Map groupMap = new HashMap(); + for (Integer index : seriesList) { + String resolutionString = String.format( + scaleFormatString, getScaleFormatStringArgs(index, 0)); + String seriesString = ""; + if (resolutionString.indexOf('/') >= 0) { + seriesString = resolutionString.substring(0, + resolutionString.lastIndexOf('/')); + } + groupMap.put(seriesString, index); + } + attributes.put("groups", groupMap); + + root.writeAttributes(attributes); + } + for (Integer index : seriesList) { try { write(index); @@ -1215,14 +1241,6 @@ public void saveResolutions(int series) sizeX, tileWidth, sizeY, tileHeight, sizeZ, chunkDepth, imageCount ); - // fileset level metadata - if (!noRootGroup) { - final ZarrGroup root = ZarrGroup.create(getRootPath()); - Map attributes = new HashMap(); - attributes.put("bioformats2raw.layout", LAYOUT); - root.writeAttributes(attributes); - } - // series level metadata setSeriesLevelMetadata(series, resolutions); diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index 437f3d5..6127a02 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -210,6 +210,13 @@ public void testAdditionalScaleFormatStringArgs() throws Exception { series0.openArray("0"); series0 = ZarrGroup.open(output.resolve("ghi/999/1").toString()); series0.openArray("0"); + + ZarrGroup z = ZarrGroup.open(output.toString()); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 2); + assertEquals(groupMap.get("abc/888/0"), 0); + assertEquals(groupMap.get("ghi/999/1"), 1); } /** @@ -240,6 +247,12 @@ public void testDefaultLayoutIsSetAndIsNested() throws Exception { ZarrGroup z = ZarrGroup.open(output.toString()); Integer layout = (Integer) z.getAttributes().get("bioformats2raw.layout"); + + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 1); + assertEquals(groupMap.get("0"), 0); + ZarrArray series0 = ZarrGroup.open(output.resolve("0")).openArray("0"); // no getter for DimensionSeparator in ZarrArray @@ -428,6 +441,12 @@ public void testMultiSeries() throws Exception { assertTool(); ZarrGroup z = ZarrGroup.open(output.toString()); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 2); + assertEquals(groupMap.get("0"), 0); + assertEquals(groupMap.get("1"), 1); + // Check series 0 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); assertArrayEquals(new int[] {1, 1, 1, 512, 512}, series0.getShape()); @@ -461,6 +480,10 @@ public void testSingleBeginningSeries() throws Exception { input = fake("series", "2"); assertTool("-s", "0"); ZarrGroup z = ZarrGroup.open(output.toString()); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 1); + assertEquals(groupMap.get("0"), 0); // Check series 0 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -488,6 +511,10 @@ public void testSingleEndSeries() throws Exception { input = fake("series", "2"); assertTool("-s", "1"); ZarrGroup z = ZarrGroup.open(output.toString()); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 1); + assertEquals(groupMap.get("0"), 1); // Check series 1 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -515,6 +542,10 @@ public void testSingleMiddleSeries() throws Exception { input = fake("series", "3"); assertTool("-s", "1"); ZarrGroup z = ZarrGroup.open(output.toString()); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 1); + assertEquals(groupMap.get("0"), 1); // Check series 1 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -911,6 +942,12 @@ public void testNoHCSOption() throws Exception { assertTool("--no-hcs"); ZarrGroup z = ZarrGroup.open(output); + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 12); + for (int i=0; i<12; i++) { + assertEquals(groupMap.get(String.valueOf(i)), i); + } // Check dimensions and block size ZarrArray series0 = z.openArray("0/0"); @@ -952,6 +989,20 @@ public void testHCSMetadata() throws Exception { int rowCount = 2; int colCount = 3; int fieldCount = 2; + + Map groupMap = + (Map) z.getAttributes().get("groups"); + assertEquals(groupMap.size(), 12); + int index = 0; + for (int r=0; r> plateMap = new HashMap>(); plateMap.put("A", Arrays.asList("1", "2", "3")); plateMap.put("B", Arrays.asList("1", "2", "3")); From beb41facbc86bfe74889153f9ce123daaaac3e1a Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 7 Jun 2022 12:28:17 -0500 Subject: [PATCH 2/3] Move group map to `OME/.zattrs` and update attribute name --- .../bioformats2raw/Converter.java | 9 +++- .../bioformats2raw/test/ZarrTest.java | 42 ++++++++++++++----- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 63b5b97..4886654 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -671,6 +671,13 @@ public void convert() Map attributes = new HashMap(); attributes.put("bioformats2raw.layout", LAYOUT); + root.writeAttributes(attributes); + } + if (!noOMEMeta) { + Path metadataPath = getRootPath().resolve("OME"); + final ZarrGroup root = ZarrGroup.create(metadataPath); + Map attributes = new HashMap(); + // record the path to each series (multiscales) and the corresponding // series (OME-XML Image) index // using the index as the key would mean that the index is stored @@ -686,7 +693,7 @@ public void convert() } groupMap.put(seriesString, index); } - attributes.put("groups", groupMap); + attributes.put("series", groupMap); root.writeAttributes(attributes); } diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index 6127a02..25a20a6 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -211,9 +211,10 @@ public void testAdditionalScaleFormatStringArgs() throws Exception { series0 = ZarrGroup.open(output.resolve("ghi/999/1").toString()); series0.openArray("0"); - ZarrGroup z = ZarrGroup.open(output.toString()); + Path omePath = output.resolve("OME"); + ZarrGroup z = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) z.getAttributes().get("series"); assertEquals(groupMap.size(), 2); assertEquals(groupMap.get("abc/888/0"), 0); assertEquals(groupMap.get("ghi/999/1"), 1); @@ -248,8 +249,10 @@ public void testDefaultLayoutIsSetAndIsNested() throws Exception { Integer layout = (Integer) z.getAttributes().get("bioformats2raw.layout"); + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); assertEquals(groupMap.get("0"), 0); @@ -441,8 +444,10 @@ public void testMultiSeries() throws Exception { assertTool(); ZarrGroup z = ZarrGroup.open(output.toString()); + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 2); assertEquals(groupMap.get("0"), 0); assertEquals(groupMap.get("1"), 1); @@ -480,8 +485,11 @@ public void testSingleBeginningSeries() throws Exception { input = fake("series", "2"); assertTool("-s", "0"); ZarrGroup z = ZarrGroup.open(output.toString()); + + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); assertEquals(groupMap.get("0"), 0); @@ -511,8 +519,11 @@ public void testSingleEndSeries() throws Exception { input = fake("series", "2"); assertTool("-s", "1"); ZarrGroup z = ZarrGroup.open(output.toString()); + + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); assertEquals(groupMap.get("0"), 1); @@ -542,8 +553,11 @@ public void testSingleMiddleSeries() throws Exception { input = fake("series", "3"); assertTool("-s", "1"); ZarrGroup z = ZarrGroup.open(output.toString()); + + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); assertEquals(groupMap.get("0"), 1); @@ -942,8 +956,11 @@ public void testNoHCSOption() throws Exception { assertTool("--no-hcs"); ZarrGroup z = ZarrGroup.open(output); + + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 12); for (int i=0; i<12; i++) { assertEquals(groupMap.get(String.valueOf(i)), i); @@ -953,7 +970,8 @@ public void testNoHCSOption() throws Exception { ZarrArray series0 = z.openArray("0/0"); assertArrayEquals(new int[] {1, 1, 1, 512, 512}, series0.getShape()); assertArrayEquals(new int[] {1, 1, 1, 512, 512}, series0.getChunks()); - assertEquals(12, z.getGroupKeys().size()); + // 12 series + OME group + assertEquals(13, z.getGroupKeys().size()); // Check OME metadata OME ome = getOMEMetadata(); @@ -990,8 +1008,10 @@ public void testHCSMetadata() throws Exception { int colCount = 3; int fieldCount = 2; + Path omePath = output.resolve("OME"); + ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); Map groupMap = - (Map) z.getAttributes().get("groups"); + (Map) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 12); int index = 0; for (int r=0; r Date: Tue, 7 Jun 2022 12:53:15 -0500 Subject: [PATCH 3/3] Switch `series` attribute from map to list --- .../bioformats2raw/Converter.java | 6 +- .../bioformats2raw/test/ZarrTest.java | 60 +++++++++++-------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 4886654..7e556dc 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -682,7 +682,7 @@ public void convert() // series (OME-XML Image) index // using the index as the key would mean that the index is stored // as a string instead of an integer - Map groupMap = new HashMap(); + List groups = new ArrayList(); for (Integer index : seriesList) { String resolutionString = String.format( scaleFormatString, getScaleFormatStringArgs(index, 0)); @@ -691,9 +691,9 @@ public void convert() seriesString = resolutionString.substring(0, resolutionString.lastIndexOf('/')); } - groupMap.put(seriesString, index); + groups.add(seriesString); } - attributes.put("series", groupMap); + attributes.put("series", groups); root.writeAttributes(attributes); } diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index 25a20a6..33a179b 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -213,11 +213,10 @@ public void testAdditionalScaleFormatStringArgs() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup z = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) z.getAttributes().get("series"); + List groupMap = (List) z.getAttributes().get("series"); assertEquals(groupMap.size(), 2); - assertEquals(groupMap.get("abc/888/0"), 0); - assertEquals(groupMap.get("ghi/999/1"), 1); + assertEquals(groupMap.get(0), "abc/888/0"); + assertEquals(groupMap.get(1), "ghi/999/1"); } /** @@ -251,10 +250,10 @@ public void testDefaultLayoutIsSetAndIsNested() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); - assertEquals(groupMap.get("0"), 0); + assertEquals(groupMap.get(0), "0"); ZarrArray series0 = ZarrGroup.open(output.resolve("0")).openArray("0"); @@ -446,11 +445,11 @@ public void testMultiSeries() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 2); - assertEquals(groupMap.get("0"), 0); - assertEquals(groupMap.get("1"), 1); + assertEquals(groupMap.get(0), "0"); + assertEquals(groupMap.get(1), "1"); // Check series 0 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -488,10 +487,13 @@ public void testSingleBeginningSeries() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); - assertEquals(groupMap.get("0"), 0); + assertEquals(groupMap.get(0), "0"); + + OME ome = getOMEMetadata(); + assertEquals(1, ome.sizeOfImageList()); // Check series 0 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -522,10 +524,13 @@ public void testSingleEndSeries() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); - assertEquals(groupMap.get("0"), 1); + assertEquals(groupMap.get(0), "0"); + + OME ome = getOMEMetadata(); + assertEquals(1, ome.sizeOfImageList()); // Check series 1 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -556,10 +561,13 @@ public void testSingleMiddleSeries() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 1); - assertEquals(groupMap.get("0"), 1); + assertEquals(groupMap.get(0), "0"); + + OME ome = getOMEMetadata(); + assertEquals(1, ome.sizeOfImageList()); // Check series 1 dimensions and special pixels ZarrArray series0 = z.openArray("0/0"); @@ -959,11 +967,11 @@ public void testNoHCSOption() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 12); for (int i=0; i<12; i++) { - assertEquals(groupMap.get(String.valueOf(i)), i); + assertEquals(groupMap.get(i), String.valueOf(i)); } // Check dimensions and block size @@ -1010,15 +1018,15 @@ public void testHCSMetadata() throws Exception { Path omePath = output.resolve("OME"); ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); - Map groupMap = - (Map) omeGroup.getAttributes().get("series"); + List groupMap = + (List) omeGroup.getAttributes().get("series"); assertEquals(groupMap.size(), 12); int index = 0; for (int r=0; r