Skip to content

Commit

Permalink
BXC-4710 remove THUMBNAIL_LARGE datastream
Browse files Browse the repository at this point in the history
  • Loading branch information
@sharonluong
sharonluong committed Sep 30, 2024
1 parent 44cab13 commit 6f14a54
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 80 deletions.
71 changes: 26 additions & 45 deletions ...ing-solr/src/test/java/edu/unc/lib/boxc/indexing/solr/filter/SetDatastreamFilterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import static edu.unc.lib.boxc.model.api.DatastreamType.JP2_ACCESS_COPY;
import static edu.unc.lib.boxc.model.api.DatastreamType.ORIGINAL_FILE;
import static edu.unc.lib.boxc.model.api.DatastreamType.TECHNICAL_METADATA;
import static edu.unc.lib.boxc.model.api.DatastreamType.THUMBNAIL_LARGE;
import static edu.unc.lib.boxc.model.fcrepo.ids.DatastreamPids.getOriginalFilePid;
import static org.apache.jena.rdf.model.ResourceFactory.createResource;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -81,11 +80,8 @@ public class SetDatastreamFilterTest {
private static final String FILE2_DIGEST = "urn:sha1:afbf62faf8a82d00969e0d4d965d62a45bb8c69b";
private static final long FILE2_SIZE = 7231l;

private static final String FILE3_MIMETYPE = "image/png";
private static final String FILE3_NAME = "image.png";
private static final String FILE3_DIGEST = "urn:sha1:280f5922b6487c39d6d01a5a8e93bfa07b8f1740";
private static final long FILE3_SIZE = 17136l;
private static final String FILE3_EXTENT = "375x250";
private static final String PNG_MIMETYPE = "image/png";
private static final String IMAGE_EXTENT = "375x250";

private static final String FILE_MP3_MIMETYPE = "audio/mpeg";
private static final String FILE_MP3_NAME = "audio.mp3";
Expand Down Expand Up @@ -121,6 +117,8 @@ public class SetDatastreamFilterTest {
private static final String PREMIS_DIGEST = "urn:sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709";
private static final long PREMIS_SIZE = 893l;

private static final long JP2_SIZE = 11;

private AutoCloseable closeable;

@TempDir
Expand Down Expand Up @@ -199,12 +197,7 @@ public void fileObjectMultipleBinariesTest() throws Exception {
fileResource(TECHNICAL_METADATA.getId(), FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST));
when(binObj2.getBinaryStream()).thenReturn(getClass().getResourceAsStream("/datastream/techmd.xml"));

BinaryObject binObj3 = mock(BinaryObject.class);
when(binObj3.getPid()).thenReturn(PIDs.get(pid.getId() + "/" + THUMBNAIL_LARGE.getId()));
when(binObj3.getResource()).thenReturn(
fileResource(THUMBNAIL_LARGE.getId(), FILE3_SIZE, FILE3_MIMETYPE, FILE3_NAME, FILE3_DIGEST));

when(fileObj.getBinaryObjects()).thenReturn(Arrays.asList(binObj, binObj2, binObj3));
when(fileObj.getBinaryObjects()).thenReturn(Arrays.asList(binObj, binObj2));
dip.setContentObject(fileObj);

filter.filter(dip);
Expand All @@ -213,50 +206,42 @@ public void fileObjectMultipleBinariesTest() throws Exception {
FILE_SIZE, FILE_MIMETYPE, FILE_NAME, FILE_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), TECHNICAL_METADATA.getId(),
FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), THUMBNAIL_LARGE.getId(),
FILE3_SIZE, FILE3_MIMETYPE, FILE3_NAME, FILE3_DIGEST, null, null);

assertEquals(FILE_SIZE, (long) idb.getFilesizeSort());
assertEquals(FILE_SIZE + FILE2_SIZE + FILE3_SIZE, (long) idb.getFilesizeTotal());
assertEquals(FILE_SIZE + FILE2_SIZE, (long) idb.getFilesizeTotal());
}

@Test
public void fileObjectImageBinaryTest() throws Exception {
when(binObj.getResource()).thenReturn(
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST));
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST));

BinaryObject binObj2 = mock(BinaryObject.class);
when(binObj2.getPid()).thenReturn(DatastreamPids.getTechnicalMetadataPid(pid));
when(binObj2.getResource()).thenReturn(
fileResource(TECHNICAL_METADATA.getId(), FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST));
when(binObj2.getBinaryStream()).thenReturn(getClass().getResourceAsStream("/datastream/techmd.xml"));

BinaryObject binObj3 = mock(BinaryObject.class);
when(binObj3.getPid()).thenReturn(PIDs.get(pid.getId() + "/" + JP2_ACCESS_COPY.getId()));
when(binObj3.getResource()).thenReturn(
fileResource(THUMBNAIL_LARGE.getId(), FILE3_SIZE, JP2_ACCESS_COPY.getMimetype(),
JP2_ACCESS_COPY.getDefaultFilename(), FILE3_DIGEST));
List<Derivative> derivs = makeJP2Derivative();
var pid = PIDs.get(PID_STRING);
when(derivativeService.getDerivatives(pid)).thenReturn(derivs);

BinaryObject binObj4 = mock(BinaryObject.class);
when(binObj4.getPid()).thenReturn(PIDs.get(pid.getId() + "/" + THUMBNAIL_LARGE.getId()));
when(binObj4.getResource()).thenReturn(
fileResource(THUMBNAIL_LARGE.getId(), FILE3_SIZE, FILE3_MIMETYPE, FILE3_NAME, FILE3_DIGEST));

when(fileObj.getBinaryObjects()).thenReturn(Arrays.asList(binObj, binObj2, binObj3, binObj4));
when(fileObj.getBinaryObjects()).thenReturn(Arrays.asList(binObj, binObj2));
dip.setContentObject(fileObj);

filter.filter(dip);

assertContainsDatastream(idb.getDatastream(), ORIGINAL_FILE.getId(),
FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST, null, FILE3_EXTENT);
FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST, null, IMAGE_EXTENT);
assertContainsDatastream(idb.getDatastream(), TECHNICAL_METADATA.getId(),
FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), THUMBNAIL_LARGE.getId(),
FILE3_SIZE, FILE3_MIMETYPE, FILE3_NAME, FILE3_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, null);

assertEquals(FILE_SIZE, (long) idb.getFilesizeSort());
// JP2 and thumbnail set to same size
assertEquals(FILE_SIZE + FILE2_SIZE + (FILE3_SIZE * 2), (long) idb.getFilesizeTotal());
assertEquals(FILE_SIZE + FILE2_SIZE + JP2_SIZE, (long) idb.getFilesizeTotal());
}

@Test
Expand Down Expand Up @@ -463,7 +448,7 @@ public void fileObjectVideoNoHeightWidthBinaryTest() throws Exception {
@Test
public void fileObjectImageBinaryNoDimensionsTest() throws Exception {
when(binObj.getResource()).thenReturn(
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST));
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST));

BinaryObject binObj2 = mock(BinaryObject.class);
when(binObj2.getPid()).thenReturn(DatastreamPids.getTechnicalMetadataPid(pid));
Expand All @@ -477,7 +462,7 @@ public void fileObjectImageBinaryNoDimensionsTest() throws Exception {
filter.filter(dip);

assertContainsDatastream(idb.getDatastream(), ORIGINAL_FILE.getId(),
FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST, null, null);
FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), TECHNICAL_METADATA.getId(),
FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST, null, null);

Expand All @@ -489,7 +474,7 @@ public void fileObjectImageBinaryNoDimensionsTest() throws Exception {
@Test
public void fileObjectImageBinaryWithJp2DimensionsTest() throws Exception {
when(binObj.getResource()).thenReturn(
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST));
fileResource(ORIGINAL_FILE.getId(), FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST));

BinaryObject binObj2 = mock(BinaryObject.class);
when(binObj2.getPid()).thenReturn(DatastreamPids.getTechnicalMetadataPid(pid));
Expand All @@ -512,11 +497,11 @@ public void fileObjectImageBinaryWithJp2DimensionsTest() throws Exception {
filter.filter(dip);

assertContainsDatastream(idb.getDatastream(), ORIGINAL_FILE.getId(),
FILE_SIZE, FILE3_MIMETYPE, "test.png", FILE_DIGEST, null, FILE3_EXTENT);
FILE_SIZE, PNG_MIMETYPE, "test.png", FILE_DIGEST, null, IMAGE_EXTENT);
assertContainsDatastream(idb.getDatastream(), TECHNICAL_METADATA.getId(),
FILE2_SIZE, FILE2_MIMETYPE, FILE2_NAME, FILE2_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
11, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, "1000x600");
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, "1000x600");

assertEquals(FILE_SIZE, (long) idb.getFilesizeSort());
}
Expand Down Expand Up @@ -623,7 +608,7 @@ public void workObjectWithThumbnailNoPrimaryObjectTest() throws Exception {
assertNotNull(idb.getFilesizeTotal());

assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
11, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, fileId, null);
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, fileId, null);
}

@Test
Expand Down Expand Up @@ -661,7 +646,7 @@ public void workObjectTestWithPrimaryAndThumbnailObjects() throws Exception {
assertEquals(FILE2_SIZE + MODS_SIZE + PREMIS_SIZE, (long) idb.getFilesizeTotal());

assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
11, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, thumbnailId, null);
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, thumbnailId, null);
}

@Test
Expand Down Expand Up @@ -691,7 +676,7 @@ public void folderObjectWithJP2Test() throws Exception {
filter.filter(dip);

assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
11, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, null);
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, null);
}

@Test
Expand All @@ -700,22 +685,18 @@ public void fileObjectWithDerivativeTest() throws Exception {
when(fileObj.getBinaryObjects()).thenReturn(List.of(binObj));
dip.setContentObject(fileObj);

File derivFile = derivDir.resolve("deriv.jp2").toFile();
FileUtils.write(derivFile, "content", "UTF-8");
long derivSize = 7l;

List<Derivative> derivs = List.of(new Derivative(JP2_ACCESS_COPY, derivFile));
List<Derivative> derivs = makeJP2Derivative();
when(derivativeService.getDerivatives(pid)).thenReturn(derivs);

filter.filter(dip);

assertContainsDatastream(idb.getDatastream(), ORIGINAL_FILE.getId(),
FILE_SIZE, FILE_MIMETYPE, FILE_NAME, FILE_DIGEST, null, null);
assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
derivSize, JP2_ACCESS_COPY.getMimetype(), derivFile.getName(), null, null, null);
JP2_SIZE, JP2_ACCESS_COPY.getMimetype(), "access.jp2", null, null, null);

assertEquals(FILE_SIZE, (long) idb.getFilesizeSort());
assertEquals(FILE_SIZE + derivSize, (long) idb.getFilesizeTotal());
assertEquals(FILE_SIZE + JP2_SIZE, (long) idb.getFilesizeTotal());
}

@Test
Expand Down
3 changes: 1 addition & 2 deletions model-api/src/main/java/edu/unc/lib/boxc/model/api/DatastreamType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ public enum DatastreamType {
MD_EVENTS("event_log", "application/n-triples", "nt", METADATA_CONTAINER, INTERNAL),
ORIGINAL_FILE("original_file", null, null, DATA_FILE_FILESET, INTERNAL),
TECHNICAL_METADATA("techmd_fits", "text/xml", "xml", DATA_FILE_FILESET, INTERNAL),
TECHNICAL_METADATA_HISTORY("techmd_fits_history", "text/xml", "xml", DATA_FILE_FILESET, INTERNAL),
THUMBNAIL_LARGE("thumbnail_large", "image/png", "png", null, EXTERNAL);
TECHNICAL_METADATA_HISTORY("techmd_fits_history", "text/xml", "xml", DATA_FILE_FILESET, INTERNAL);

private final String id;
private final String mimetype;
Expand Down
2 changes: 1 addition & 1 deletion search-solr/src/test/java/edu/unc/lib/boxc/search/solr/filters/QueryFilterFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
public class QueryFilterFactoryTest {
@Test
public void NamedDatastreamFilterTest() {
var filter = QueryFilterFactory.createFilter(SearchFieldKey.DATASTREAM, DatastreamType.THUMBNAIL_LARGE);
var filter = QueryFilterFactory.createFilter(SearchFieldKey.DATASTREAM, DatastreamType.JP2_ACCESS_COPY);
assertInstanceOf(NamedDatastreamFilter.class, filter);
}

Expand Down
25 changes: 0 additions & 25 deletions ...a/edu/unc/lib/boxc/services/camel/destroyDerivatives/DestroyDerivativesProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import static edu.unc.lib.boxc.model.api.DatastreamType.FULLTEXT_EXTRACTION;
import static edu.unc.lib.boxc.model.api.DatastreamType.JP2_ACCESS_COPY;
import static edu.unc.lib.boxc.model.api.DatastreamType.THUMBNAIL_LARGE;
import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrBinaryMimeType;
import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrBinaryPidId;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down Expand Up @@ -97,30 +96,6 @@ public void deleteFulltextTest() throws Exception {
assertTrue(derivativeTypeBaseDir.exists());
}

@Test
public void deleteThumbnailTest() throws Exception {
derivativeTypeDir = THUMBNAIL_LARGE.getId();
derivativeFinalDir = Files.createDirectories(tmpFolder.resolve(derivativeTypeDir + "/de/75/d8/11")).toFile();
file = new File(derivativeFinalDir, pathId + ".png");

FileUtils.writeStringToFile(file, "fake image", StandardCharsets.UTF_8);

derivativeTypeBaseDir = new File(derivativeDirBase, derivativeTypeDir);
processor = new DestroyDerivativesProcessor("png", derivativeTypeBaseDir.getAbsolutePath());

when(message.getHeader(eq(CdrBinaryMimeType)))
.thenReturn("image/png");

processor.process(exchange);

// Deleted file
assertFalse(file.exists());
// Deleted parent dirs
assertFalse(new File(derivativeTypeBaseDir, "de").exists());
// Didn't delete root derivative type dir
assertTrue(derivativeTypeBaseDir.exists());
}

@Test
public void deleteCollectionSrcImgTest() throws Exception {
String srcDirBase = "srcDir";
Expand Down
11 changes: 4 additions & 7 deletions web-common/src/test/java/edu/unc/lib/boxc/web/common/services/AccessCopiesServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,8 @@ private ContentObjectSolrRecord createImgObject(ResourceType resourceType) {
var mdObjectImg = new ContentObjectSolrRecord();
mdObjectImg.setResourceType(resourceType.name());
mdObjectImg.setId(UUID.randomUUID().toString());
List<String> imgDatastreams = Arrays.asList(
ORIGINAL_FILE.getId() + "|image/png|file.png|png|766|urn:sha1:checksum|",
DatastreamType.THUMBNAIL_LARGE.getId() + "|image/png|thumb|png|55||",
DatastreamType.JP2_ACCESS_COPY.getId() + "|image/jp2|thumb|jp2|555||");
List<String> imgDatastreams = List.of(
ORIGINAL_FILE.getId() + "|image/png|file.png|png|766|urn:sha1:checksum|");
mdObjectImg.setFileFormatCategory(Collections.singletonList(ContentCategory.image.getDisplayName()));
mdObjectImg.setFileFormatType(Collections.singletonList("image/png"));
mdObjectImg.setDatastream(imgDatastreams);
Expand Down Expand Up @@ -292,9 +290,8 @@ public void getThumbnailIdNoPrimaryMultipleImages() {
var mdObjectImg2 = new ContentObjectSolrRecord();
mdObjectImg2.setResourceType(ResourceType.File.name());
mdObjectImg2.setId(UUID.randomUUID().toString());
var imgDatastreams = Arrays.asList(
ORIGINAL_FILE.getId() + "|image/jpg|file2.png|png|555|urn:sha1:checksum|",
DatastreamType.THUMBNAIL_LARGE.getId() + "|image/png|thumb|png|55||");
var imgDatastreams = List.of(
ORIGINAL_FILE.getId() + "|image/jpg|file2.png|png|555|urn:sha1:checksum|");
mdObjectImg2.setFileFormatCategory(Collections.singletonList(ContentCategory.image.getDisplayName()));
mdObjectImg2.setFileFormatType(Collections.singletonList("png"));
mdObjectImg2.setDatastream(imgDatastreams);
Expand Down

0 comments on commit 6f14a54

Please sign in to comment.