BXC-4510 add StreamingMetadataService (#89)

* add StreamingMetadataService and streaming metadata to test resources * Fix test to account for extra columns. Update matcher due to mockito 5 * DRY up query execution, cache whether project has streaming fields, make return type more specific, throw explicit error when no fields * Update counts based on adding streaming columns --------- Co-authored-by: Ben Pennell <[email protected]>
UNC-Libraries · Apr 12, 2024 · 1a97967 · 1a97967
1 parent ea246f2
commit 1a97967
Show file tree

Hide file tree

Showing 7 changed files with 221 additions and 3 deletions.
diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java
@@ -0,0 +1,130 @@
+package edu.unc.lib.boxc.migration.cdm.services;
+
+import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
+import edu.unc.lib.boxc.migration.cdm.model.CdmFieldInfo;
+import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+
+import static edu.unc.lib.boxc.migration.cdm.util.CLIConstants.outputLogger;
+
+/**
+ * Service for retrieving streaming metadata
+ * @author krwong
+ */
+public class StreamingMetadataService {
+    private static final Logger log = LoggerFactory.getLogger(StreamingMetadataService.class);
+
+    private MigrationProject project;
+    private CdmFieldService fieldService;
+    private CdmIndexService indexService;
+
+    public static final String STREAMING_FILE_FIELD = "stream";
+    public static final String DURACLOUD_SPACE_FIELD = "duracl";
+    public static final String PLAYLIST_FILE_EXTENSION = "-playlist.m3u8";
+    public static final String DURACLOUD_OPEN = "open-hls";
+    public static final String DURACLOUD_CAMPUS = "campus-hls";
+    public static final String DURACLOUD_CLOSED = "closed-hls";
+    public static final String STREAMING_HOST = "duracloud";
+
+    private Boolean projectHasStreamingMetadata = null;
+
+    /**
+     * Verify if a record has streaming metadata
+     * @param cdmId
+     * @return true/false
+     */
+    public boolean verifyRecordHasStreamingMetadata(String cdmId) {
+        if (!hasProjectStreamingMetadataField()) {
+            return false;
+        }
+
+        var streamingFields = getStreamingFieldValues(cdmId);
+        return streamingFields[0] != null && streamingFields[1] != null;
+    }
+
+    private boolean hasProjectStreamingMetadataField() {
+        if (projectHasStreamingMetadata == null) {
+            // check if project has streamingFile field and duracloudSpace field
+            fieldService.validateFieldsFile(project);
+            CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project);
+            List<String> exportFields = fieldInfo.listAllExportFields();
+            projectHasStreamingMetadata = exportFields.contains(STREAMING_FILE_FIELD) && exportFields.contains(DURACLOUD_SPACE_FIELD);
+        }
+        return projectHasStreamingMetadata;
+    }
+
+    /**
+     * Retrieve streaming metadata and remap to correct values
+     * @param cdmId
+     * @return object with streamingFile, duracloudSpace, and streamingHost fields
+     */
+    public String[] getStreamingMetadata(String cdmId) {
+        var streamingValues = getStreamingFieldValues(cdmId);
+        String duracloudSpace = streamingValues[1];
+        String streamingFile = streamingValues[0];
+
+        if (duracloudSpace == null || streamingFile == null) {
+            throw new MigrationException("Streaming metadata not found for " + cdmId);
+        }
+        // transform to playlist file extensions
+        streamingFile = streamingFile.split("\\.")[0] + PLAYLIST_FILE_EXTENSION;
+
+        // transform to current duracloud space IDs
+        if (duracloudSpace.contains("open")) {
+            duracloudSpace = DURACLOUD_OPEN;
+        } else if (duracloudSpace.contains("campus")) {
+            duracloudSpace = DURACLOUD_CAMPUS;
+        } else if (duracloudSpace.contains("closed")) {
+            duracloudSpace = DURACLOUD_CLOSED;
+        }
+
+        return new String[] {streamingFile, duracloudSpace, STREAMING_HOST};
+    }
+
+    private String[] getStreamingFieldValues(String cdmId) {
+        String duracloudSpace = null;
+        String streamingFile = null;
+
+        // retrieve streaming metadata
+        Connection conn = null;
+        try {
+            conn = indexService.openDbConnection();
+            Statement stmt = conn.createStatement();
+            ResultSet rs = stmt.executeQuery("select " + STREAMING_FILE_FIELD + ", " + DURACLOUD_SPACE_FIELD
+                    + " from " + CdmIndexService.TB_NAME
+                    + " where " + CdmFieldInfo.CDM_ID + " = " + cdmId);
+            while (rs.next()) {
+                if (!rs.getString(1).isEmpty()) {
+                    streamingFile = rs.getString(1);
+                }
+                if (!rs.getString(2).isEmpty()) {
+                    duracloudSpace = rs.getString(2);
+                }
+            }
+        } catch (SQLException e) {
+            throw new MigrationException("Error interacting with export index", e);
+        } finally {
+            CdmIndexService.closeDbConnection(conn);
+        }
+        return new String[] {streamingFile, duracloudSpace};
+    }
+
+    public void setProject(MigrationProject project) {
+        this.project = project;
+    }
+
+    public void setFieldService(CdmFieldService fieldService) {
+        this.fieldService = fieldService;
+    }
+
+    public void setIndexService(CdmIndexService indexService) {
+        this.indexService = indexService;
+    }
+}
diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java
@@ -54,7 +54,7 @@ public void reportInitialized() throws Exception {
 
         assertOutputContains("CDM Collection Fields");
         assertOutputMatches(".*Mapping File Valid: +Yes.*");
-        assertOutputMatches(".*Fields: +61\n.*");
+        assertOutputMatches(".*Fields: +63\n.*");
         assertOutputMatches(".*Skipped: +1\n.*");
 
         assertOutputContains("CDM Collection Exports");

diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java
@@ -3,7 +3,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
-import static org.mockito.Matchers.any;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 import static org.mockito.MockitoAnnotations.openMocks;
 

diff --git a/...test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java b/...test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java
@@ -53,7 +53,7 @@ public void allExpectedCellsPopulatedTest() throws Exception {
         XSSFWorkbook workbook = new XSSFWorkbook(inputStream);
         Sheet sheet = workbook.getSheetAt(0);
 
-        assertEquals(60, sheet.getLastRowNum());
+        assertEquals(62, sheet.getLastRowNum());
         assertEquals(16, sheet.getRow(0).getPhysicalNumberOfCells());
         assertEquals(12, sheet.getRow(1).getPhysicalNumberOfCells());
         assertEquals(12, sheet.getRow(60).getPhysicalNumberOfCells());

diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java
@@ -0,0 +1,84 @@
+package edu.unc.lib.boxc.migration.cdm.services;
+
+import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
+import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
+import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
+import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.MockitoAnnotations.openMocks;
+
+public class StreamingMetadataServiceTest {
+    private static final String PROJECT_NAME = "proj";
+
+    @TempDir
+    public Path tmpFolder;
+
+    private SipServiceHelper testHelper;
+    private MigrationProject project;
+    private StreamingMetadataService service;
+    private AutoCloseable closeable;
+
+    @BeforeEach
+    public void setup() throws Exception {
+        closeable = openMocks(this);
+        project = MigrationProjectFactory.createMigrationProject(
+                tmpFolder, PROJECT_NAME, null, "user",
+                CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
+        testHelper = new SipServiceHelper(project, tmpFolder);
+        service = new StreamingMetadataService();
+        service.setProject(project);
+        service.setFieldService(testHelper.getFieldService());
+        service.setIndexService(testHelper.getIndexService());
+    }
+
+    @AfterEach
+    void closeService() throws Exception {
+        closeable.close();
+    }
+
+    @Test
+    public void verifyNoStreamingMetadata() throws Exception {
+        testHelper.indexExportData("mini_gilmer");
+
+        var result = service.verifyRecordHasStreamingMetadata("25");
+        assertFalse(result);
+    }
+
+    @Test
+    public void verifyHasStreamingMetadata() throws Exception {
+        testHelper.indexExportData("mini_gilmer");
+
+        var result = service.verifyRecordHasStreamingMetadata("27");
+        assertTrue(result);
+    }
+
+    @Test
+    public void getStreamingMetadataSuccess() throws Exception {
+        testHelper.indexExportData("mini_gilmer");
+
+        var result = service.getStreamingMetadata("27");
+        assertEquals("gilmer_recording-playlist.m3u8", result[0]);
+        assertEquals("open-hls", result[1]);
+        assertEquals("duracloud", result[2]);
+    }
+
+    @Test
+    public void getStreamingMetadataFail() throws Exception {
+        testHelper.indexExportData("mini_gilmer");
+
+        Exception exception = assertThrows(MigrationException.class, () -> {
+            service.getStreamingMetadata("25");
+        });
+    }
+}
diff --git a/src/test/resources/descriptions/mini_gilmer/index/description/desc.all b/src/test/resources/descriptions/mini_gilmer/index/description/desc.all
@@ -182,6 +182,8 @@
 <fila>TIFF</fila>
 <filb>TIFF</filb>
 <groupa>group2</groupa>
+<stream>gilmer_recording.mp3</stream>
+<duracl>sfc20009-open</duracl>
 <full>/shc/gilmer_maps/</full>
 <fullrs>276_203_E.tif</fullrs>
 <find>50.jp2</find>

diff --git a/src/test/resources/gilmer_fields.csv b/src/test/resources/gilmer_fields.csv
@@ -52,6 +52,8 @@ color,color,Color Space Raw Scan,false,n,n,y,n,BLANK
 coloa,coloa,Color Space filename,false,n,n,y,n,BLANK
 fila,fila,File Format Raw Scan,false,n,n,y,y,BLANK
 filb,filb,File Format filename,false,n,n,y,y,BLANK
+stream,stream,StreamingFile,false,n,n,n,y,BLANK
+duracl,duracl,duracloudSpace,false,n,n,n,y,identi
 full,full,path,false,n,n,n,n,BLANK
 fullrs,fullrs,Full resolution,false,n,n,y,n,
 dmoclcno,dmoclcno,OCLC number,false,n,n,y,n,