BXC-4712 update enhancements for thumbnails (#1793)

* BXC-4712 update enhancements for thumbnails

* BXC-4712 remove unused constant

* BXC-4712 adding new request and processors and updating thumbnail router

* BXC-4712 update tsts and message senders

* BXC-4712 fix router test

* BXC-4712 remove unused variable

* BXC-4712 cleanup

* extra space removed

* BXC-4712 make separate queues for importing thumbnails

* BXC-4712 fix import thumbnail router test

* BXC-4712 update URI to string

* BXC-4712 update storage path to string

* BXC-4712 update thumbnail datatream choice

* BXC-4712 add datastream test

* BXC-4712 a bit of cleanup

* BXC-4712 working on datastream controller IT

* BXC-4712 adjusting download image service

* BXC-4712 set up test corpus datastreams correctly

* BXC-4712 trigger indexing

* BXC-4712 fix tests

* BXC-4712 cleanup imports

indexing-solr/src/main/java/edu/unc/lib/boxc/indexing/solr/filter/SetDatastreamFilter.java

@@ -334,7 +334,8 @@ private long getFilesize(FileObject fileObject, List<Datastream> datastreams) th
     }
 
     private void addDerivatives(List<Datastream> dsList, PID pid, boolean ownedByOtherObject, List<DatastreamType> types) {
-        derivativeService.getDerivatives(pid).forEach(deriv -> {
+        var derivatives = derivativeService.getDerivatives(pid);
+        derivatives.forEach(deriv -> {
             DatastreamType type = deriv.getType();
             // only add derivatives of types listed
             if ((types != null) && !types.contains(type)) {

indexing-solr/src/test/java/edu/unc/lib/boxc/indexing/solr/filter/SetDatastreamFilterTest.java

@@ -691,6 +691,24 @@ public void folderObjectWithMetadataTest() throws Exception {
         assertEquals(FILE2_SIZE + MODS_SIZE + PREMIS_SIZE, (long) idb.getFilesizeTotal());
     }
 
+    @Test
+    public void folderObjectWithJP2Test() throws Exception {
+        FolderObject folderObj = mock(FolderObject.class);
+        when(folderObj.getPid()).thenReturn(pid);
+
+        File file = derivDir.resolve("small.jp2").toFile();
+        FileUtils.write(file, "content", "UTF-8");
+        List<Derivative> derivs = List.of(new Derivative(JP2_ACCESS_COPY, file));
+        when(derivativeService.getDerivatives(pid)).thenReturn(derivs);
+
+        dip.setContentObject(folderObj);
+
+        filter.filter(dip);
+
+        assertContainsDatastream(idb.getDatastream(), JP2_ACCESS_COPY.getId(),
+                7l, JP2_ACCESS_COPY.getMimetype(), "small.jp2", null, null, null);
+    }
+
     @Test
     public void fileObjectWithDerivativeTest() throws Exception {
         when(fileObj.getPid()).thenReturn(pid);

indexing-solr/src/test/java/edu/unc/lib/boxc/indexing/solr/test/TestCorpus.java

@@ -78,6 +78,9 @@ public List<SolrInputDocument> populate() {
         newDoc.addField("ancestorIds", makeAncestorIds(pid1, pid2));
         newDoc.addField("ancestorPath", makeAncestorPath(pid1));
         newDoc.addField("resourceType", "Collection");
+        List<String> collectionDatastream = List.of(
+                DatastreamType.JP2_ACCESS_COPY.getId() + "|image/jp2|bunny.jp2|jp2|||" + pid2.getId() + "|1200x1200");
+        newDoc.addField(SearchFieldKey.DATASTREAM.getSolrField(), collectionDatastream);
         docs.add(newDoc);
 
         newDoc = new SolrInputDocument();
@@ -115,8 +118,8 @@ public List<SolrInputDocument> populate() {
         newDoc.addField("ancestorPath", makeAncestorPath(pid1, pid2, pid6));
         newDoc.addField("resourceType", ResourceType.File.name());
         List<String> imgDatastreams = Arrays.asList(
-                ORIGINAL_FILE.getId() + "|image/png|file.png|png|766|urn:sha1:checksum|1200x1200",
-                DatastreamType.JP2_ACCESS_COPY.getId() + "|image/jp2|bunny.jp2|jp2|||1200x1200");
+                ORIGINAL_FILE.getId() + "|image/png|file.png|png|766|urn:sha1:checksum||1200x1200",
+                DatastreamType.JP2_ACCESS_COPY.getId() + "|image/jp2|bunny.jp2|jp2|||" + pid6File.getId() + "|1200x1200");
         newDoc.addField(SearchFieldKey.DATASTREAM.getSolrField(), imgDatastreams);
         newDoc.addField(SearchFieldKey.FILE_FORMAT_CATEGORY.getSolrField(), ContentCategory.image.getDisplayName());
         newDoc.addField(SearchFieldKey.FILE_FORMAT_TYPE.getSolrField(), "png");

integration/src/test/java/edu/unc/lib/boxc/integration/factories/ContentObjectFactory.java

@@ -62,7 +62,7 @@ public Map<String, String> validateOptions(Map<String, String> options) {
     }
 
     protected void addThumbnail(ContentObject object) throws IOException {
-        var derivativePath = derivativeService.getDerivativePath(object.getPid(), DatastreamType.THUMBNAIL_LARGE);
+        var derivativePath = derivativeService.getDerivativePath(object.getPid(), DatastreamType.JP2_ACCESS_COPY);
         FileUtils.write(derivativePath.toFile(), "image", "UTF-8");
     }
 

it_config/allowed_external_paths.txt

@@ -1 +1 @@
-file:/tmp/boxc_test_storage/
+file:/tmp/boxc_test_storage/

operations-jms/src/main/java/edu/unc/lib/boxc/operations/jms/thumbnails/ImportThumbnailRequest.java

@@ -0,0 +1,52 @@
+package edu.unc.lib.boxc.operations.jms.thumbnails;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import edu.unc.lib.boxc.auth.api.models.AgentPrincipals;
+import edu.unc.lib.boxc.auth.fcrepo.models.AgentPrincipalsImpl;
+
+import java.nio.file.Path;
+
+/**
+ * Request object for importing an image as thumbnail for a collection, folder, or admin unit
+ *
+ * @author snluong
+ */
+public class ImportThumbnailRequest {
+    @JsonDeserialize(as = AgentPrincipalsImpl.class)
+    private AgentPrincipals agent;
+    private String mimetype;
+    private Path storagePath;
+    private String pidString;
+
+    public String getPidString() {
+        return pidString;
+    }
+
+    public void setPidString(String pidString) {
+        this.pidString = pidString;
+    }
+
+    public AgentPrincipals getAgent() {
+        return agent;
+    }
+
+    public void setAgent(AgentPrincipals agent) {
+        this.agent = agent;
+    }
+
+    public String getMimetype() {
+        return mimetype;
+    }
+
+    public void setMimetype(String mimetype) {
+        this.mimetype = mimetype;
+    }
+
+    public Path getStoragePath() {
+        return storagePath;
+    }
+
+    public void setStoragePath(Path storagePath) {
+        this.storagePath = storagePath;
+    }
+}

operations-jms/src/main/java/edu/unc/lib/boxc/operations/jms/thumbnails/ImportThumbnailRequestSerializationHelper.java

@@ -0,0 +1,46 @@
+package edu.unc.lib.boxc.operations.jms.thumbnails;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+import java.io.IOException;
+
+/**
+ * Helper methods for serializing and deserializing import thumbnail requests
+ *
+ * @author snluong
+ */
+public class ImportThumbnailRequestSerializationHelper {
+    private static final ObjectWriter REQUEST_WRITER;
+    private static final ObjectReader REQUEST_READER;
+    static {
+        ObjectMapper mapper = new ObjectMapper();
+        REQUEST_WRITER = mapper.writerFor(ImportThumbnailRequest.class);
+        REQUEST_READER = mapper.readerFor(ImportThumbnailRequest.class);
+    }
+
+    private ImportThumbnailRequestSerializationHelper() {
+    }
+
+    /**
+     * Transform request into a JSON string
+     * @param request
+     * @return
+     * @throws IOException
+     */
+    public static String toJson(ImportThumbnailRequest request) throws IOException {
+        return REQUEST_WRITER.writeValueAsString(request);
+    }
+
+    /**
+     * Transform JSON string to an ImportThumbnailRequest
+     * @param json
+     * @return
+     * @throws IOException
+     */
+    public static ImportThumbnailRequest toRequest(String json) throws IOException {
+        return REQUEST_READER.readValue(json);
+    }
+}
+

operations-jms/src/main/java/edu/unc/lib/boxc/operations/jms/thumbnails/ThumbnailRequestSender.java

@@ -16,6 +16,8 @@
 public class ThumbnailRequestSender extends MessageSender {
     private static final Logger log = LoggerFactory.getLogger(ThumbnailRequestSender.class);
     private static final ObjectWriter MAPPER = new ObjectMapper().writerFor(ThumbnailRequest.class);
+    private static final ObjectWriter IMPORT_MAPPER = new ObjectMapper().writerFor(ImportThumbnailRequest.class);
+    private String importDestinationName;
 
     /**
      * Send a ThumbnailRequest to the configured JMS queue
@@ -28,4 +30,30 @@ public void sendToQueue(ThumbnailRequest request) throws IOException {
         log.info("Job to {} thumbnail has been queued for {} with file {}",
                 request.getAction(), request.getAgent().getUsername(), request.getFilePidString());
     }
+
+    /**
+     * Send an ImportThumbnailRequest to the appropriate JMS queue
+     * @param request
+     * @throws IOException
+     */
+    public void sendToImportQueue(ImportThumbnailRequest request) throws IOException {
+        String messageBody = IMPORT_MAPPER.writeValueAsString(request);
+        sendImportMessage(messageBody);
+        log.info("Job to import thumbnail has been queued for {} with object {}",
+                request.getAgent().getUsername(), request.getPidString());
+    }
+
+    public void sendImportMessage(String msgStr) {
+        jmsTemplate.send(importDestinationName, (session -> {
+            // Committing the session to flush changes in long-running threads
+            if (session.getTransacted()) {
+                session.commit();
+            }
+            return session.createTextMessage(msgStr);
+        }));
+    }
+
+    public void setImportDestinationName(String importDestinationName) {
+        this.importDestinationName = importDestinationName;
+    }
 }

operations-jms/src/test/java/edu/unc/lib/boxc/operations/jms/thumbnails/ThumbnailRequestSenderTest.java

@@ -8,13 +8,16 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.mockito.Mock;
 import org.springframework.jms.core.JmsTemplate;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.UUID;
 
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.verify;
 import static org.mockito.MockitoAnnotations.openMocks;
 
@@ -24,15 +27,19 @@
 public class ThumbnailRequestSenderTest {
     @Mock
     private JmsTemplate jmsTemplate;
+    @TempDir
+    private Path storagePath;
     private ThumbnailRequestSender thumbnailRequestSender;
     private AutoCloseable closeable;
     private final AgentPrincipals agent = new AgentPrincipalsImpl("user", new AccessGroupSetImpl("agroup"));
+    private String importQueue = "import.queue";
 
     @BeforeEach
     public void setup() {
         closeable = openMocks(this);
         thumbnailRequestSender = new ThumbnailRequestSender();
         thumbnailRequestSender.setJmsTemplate(jmsTemplate);
+        thumbnailRequestSender.setImportDestinationName(importQueue);
     }
 
     @AfterEach
@@ -52,6 +59,19 @@ public void sendToQueueTest() throws IOException {
         verify(jmsTemplate).send(any());
     }
 
+    @Test
+    public void sendToImportQueueTest() throws IOException {
+        var pidString = makePid().toString();
+        var request = new ImportThumbnailRequest();
+        request.setAgent(agent);
+        request.setPidString(pidString);
+        request.setStoragePath(storagePath);
+        request.setMimetype("image/jpeg");
+
+        thumbnailRequestSender.sendToImportQueue(request);
+        verify(jmsTemplate).send(eq(importQueue), any());
+    }
+
     public static PID makePid() {
         return PIDs.get(UUID.randomUUID().toString());
     }

services-camel-app/src/main/java/edu/unc/lib/boxc/services/camel/enhancements/EnhancementRouter.java

@@ -42,7 +42,6 @@ public class EnhancementRouter extends RouteBuilder {
     private Integer enhancementThreads;
 
     private static final String DEFAULT_ENHANCEMENTS = "thumbnails,imageAccessCopy,extractFulltext";
-    private static final String THUMBNAIL_ENHANCEMENTS = "thumbnails";
     @Override
     public void configure() throws Exception {
 
@@ -69,7 +68,6 @@ public void configure() throws Exception {
                         ))
                     .log(DEBUG, log, "Processing enhancements for non-binary ${headers[CamelFcrepoUri]}")
                     .process(nbProcessor)
-                    .setHeader(CdrEnhancementSet, constant(THUMBNAIL_ENHANCEMENTS))
                     .to("{{cdr.enhancement.perform.camel}}")
                 .otherwise()
                     .log(DEBUG, log, "Ignoring resource ${headers[CamelFcrepoUri]}")

services-camel-app/src/main/java/edu/unc/lib/boxc/services/camel/thumbnails/ImportThumbnailRequestProcessor.java

@@ -0,0 +1,33 @@
+package edu.unc.lib.boxc.services.camel.thumbnails;
+
+import edu.unc.lib.boxc.model.fcrepo.ids.PIDs;
+import edu.unc.lib.boxc.operations.jms.thumbnails.ImportThumbnailRequestSerializationHelper;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+
+import java.io.IOException;
+
+import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrBinaryMimeType;
+import static edu.unc.lib.boxc.services.camel.util.CdrFcrepoHeaders.CdrBinaryPath;
+import static org.fcrepo.camel.FcrepoHeaders.FCREPO_URI;
+
+/**
+ * Processing requests to import images to use as a thumbnail for a non-work Repository object
+ *
+ * @author snluong
+ */
+public class ImportThumbnailRequestProcessor implements Processor {
+    @Override
+    public void process(Exchange exchange) throws IOException {
+        var in = exchange.getIn();
+        var request = ImportThumbnailRequestSerializationHelper.toRequest(in.getBody(String.class));
+        var mimetype = request.getMimetype();
+        var storagePath = request.getStoragePath();
+        var pidString = request.getPidString();
+        var repoPath = PIDs.get(pidString).getRepositoryPath();
+
+        in.setHeader(CdrBinaryPath, storagePath.toString());
+        in.setHeader(CdrBinaryMimeType, mimetype);
+        in.setHeader(FCREPO_URI, repoPath);
+    }
+}

services-camel-app/src/main/java/edu/unc/lib/boxc/services/camel/thumbnails/ThumbnailRouter.java

@@ -8,20 +8,33 @@
 import static org.slf4j.LoggerFactory.getLogger;
 
 /**
- * Router for processing assigning thumbnails for works
+ * Router for processing thumbnails:
+ * Importing thumbnails for collections, folders, and admin units
+ * Assigning thumbnails for works
  *
  * @author snluong
  */
 public class ThumbnailRouter extends RouteBuilder {
     private static final Logger log = getLogger(ThumbnailRouter.class);
     @BeanInject(value = "thumbnailRequestProcessor")
     private ThumbnailRequestProcessor thumbnailRequestProcessor;
+    @BeanInject(value = "importThumbnailRequestProcessor")
+    private ImportThumbnailRequestProcessor importThumbnailRequestProcessor;
 
     @Override
     public void configure() throws Exception {
         from("{{cdr.thumbnails.stream.camel}}")
-                .routeId("DcrThumbnails")
-                .log(DEBUG, log, "Received thumbnail request")
-                .bean(thumbnailRequestProcessor);
+            .routeId("DcrThumbnails")
+            .log(DEBUG, log,
+                    "Received thumbnail request: assigning thumbnail for ${headers[CamelFcrepoUri]}")
+            .bean(thumbnailRequestProcessor);
+
+        from("{{cdr.import.thumbnails.stream.camel}}")
+                .routeId("DcrImportThumbnails")
+                .process(importThumbnailRequestProcessor)
+                .log(DEBUG, log,
+                        "Received thumbnail request: importing thumbnail for ${headers[CamelFcrepoUri]}")
+                // trigger JP2 generation sequentially followed by indexing
+                .to("direct:process.enhancement.imageAccessCopy", "direct:solrIndexing");
     }
 }

services-camel-app/src/main/webapp/WEB-INF/service-context.xml

@@ -517,6 +517,9 @@
         <property name="repositoryObjectFactory" ref="repositoryObjectFactory" />
     </bean>
 
+    <bean id="importThumbnailRequestProcessor" class="edu.unc.lib.boxc.services.camel.thumbnails.ImportThumbnailRequestProcessor">
+    </bean>
+
     <bean id="viewSettingRequestProcessor" class="edu.unc.lib.boxc.services.camel.viewSettings.ViewSettingRequestProcessor">
         <property name="accessControlService" ref="aclService" />
         <property name="repositoryObjectLoader" ref="repositoryObjectLoader" />

services-camel-app/src/test/java/edu/unc/lib/boxc/services/camel/enhancements/EnhancementRouterIT.java

@@ -166,8 +166,8 @@ public void nonBinaryWithSourceImages() throws Exception {
                 Cdr.Collection.getURI(), Container.getURI());
         template.sendBodyAndHeaders("", headers);
 
-        verify(addSmallThumbnailProcessor, timeout(ALLOW_WAIT)).process(any(Exchange.class));
-        verify(addLargeThumbnailProcessor, timeout(ALLOW_WAIT)).process(any(Exchange.class));
+        verify(addSmallThumbnailProcessor, never()).process(any(Exchange.class));
+        verify(addLargeThumbnailProcessor, never()).process(any(Exchange.class));
         verify(addAccessCopyProcessor, never()).process(any(Exchange.class));
         verify(solrIngestProcessor, timeout(ALLOW_WAIT)).process(any(Exchange.class));
     }