Add job for performing velocicroptor action (#108)

src/main/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJob.java

@@ -0,0 +1,118 @@
+package edu.unc.lib.boxc.migration.cdm.jobs;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
+import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
+import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions;
+import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService;
+import edu.unc.lib.boxc.migration.cdm.util.SshClientService;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Job which prepares and executes a remote velocicroptor job to crop color bars from images
+ * @author bbpennel
+ */
+public class VelocicroptorRemoteJob {
+    protected static final String RESULTS_REL_PATH = "processing/results/velocicroptor";
+    private static final String JOB_ID_PATTERN_FORMAT = "ddMMyyyyHHmmssSSS";
+    private static final DateTimeFormatter JOB_ID_FORMATTER = DateTimeFormatter.ofPattern(JOB_ID_PATTERN_FORMAT)
+            .withZone(ZoneId.systemDefault());
+
+    private SshClientService sshClientService;
+    private MigrationProject project;
+    private SourceFilesToRemoteService sourceFilesToRemoteService;
+    private Path remoteProjectsPath;
+    private Path remoteJobScriptPath;
+    private String adminEmail;
+    private String outputServer;
+    private Path outputPath;
+
+    /**
+     * Perform the velocicroptor job on the source files for the project
+     * @param options options for the job
+     * @return id of the job that was performed
+     */
+    public String run(ProcessSourceFilesOptions options) {
+        // Generate job id
+        var startInstant = Instant.now();
+        var jobId = JOB_ID_FORMATTER.format(startInstant);
+
+        // Create local results directory
+        var resultsPath = project.getProjectPath().resolve(RESULTS_REL_PATH);
+        try {
+            Files.createDirectories(resultsPath);
+
+            var remoteJobPath = remoteProjectsPath.resolve(project.getProjectName() + "/velocicroptor/" + jobId);
+            // Transfer source files to remote server
+            var remoteSourcesPath = remoteJobPath.resolve("source_files");
+            sourceFilesToRemoteService.transferFiles(remoteSourcesPath);
+
+            // Create remote job execution config file
+            ObjectMapper mapper = new ObjectMapper();
+            var config = createJobConfig(options, startInstant, jobId);
+            String configJson = mapper.writeValueAsString(config);
+
+            // Trigger remote job, passing config as argument
+            sshClientService.executeRemoteCommand("sbatch " + remoteJobScriptPath.toString() + " '" + configJson + "'");
+        } catch (IOException e) {
+            throw new MigrationException(e);
+        }
+        return jobId;
+    }
+
+    private Map<String, String> createJobConfig(ProcessSourceFilesOptions options, Instant startInstant, String jobId) {
+        Map<String, String> config = new HashMap<>();
+        config.put("job_id", jobId);
+        config.put("job_name", options.getActionName());
+        config.put("chompb_proj_name", project.getProjectName());
+        config.put("admin_address", adminEmail);
+        // User that initiated the job
+        config.put("username", options.getUsername());
+        config.put("email_address", options.getEmailAddress());
+        config.put("start_time", startInstant.toString());
+        // Details about where the job should send the results once it has completed
+        config.put("output_path", outputPath.toString());
+        config.put("output_server", outputServer);
+        return config;
+    }
+
+    public void setSshClientService(SshClientService sshClientService) {
+        this.sshClientService = sshClientService;
+    }
+
+    public void setProject(MigrationProject project) {
+        this.project = project;
+    }
+
+    public void setSourceFilesToRemoteService(SourceFilesToRemoteService sourceFilesToRemoteService) {
+        this.sourceFilesToRemoteService = sourceFilesToRemoteService;
+    }
+
+    public void setRemoteProjectsPath(Path remoteProjectsPath) {
+        this.remoteProjectsPath = remoteProjectsPath;
+    }
+
+    public void setRemoteJobScriptPath(Path remoteJobScriptPath) {
+        this.remoteJobScriptPath = remoteJobScriptPath;
+    }
+
+    public void setAdminEmail(String adminEmail) {
+        this.adminEmail = adminEmail;
+    }
+
+    public void setOutputServer(String outputServer) {
+        this.outputServer = outputServer;
+    }
+
+    public void setOutputPath(Path outputPath) {
+        this.outputPath = outputPath;
+    }
+}

src/main/java/edu/unc/lib/boxc/migration/cdm/options/ProcessSourceFilesOptions.java

@@ -0,0 +1,46 @@
+package edu.unc.lib.boxc.migration.cdm.options;
+
+import picocli.CommandLine;
+
+/**
+ * Options for job to process source files
+ * @author bbpennel
+ */
+public class ProcessSourceFilesOptions {
+    @CommandLine.Option(names = {"-a", "--action"},
+            description = "Name of the processing action to execute.")
+    private String actionName;
+
+    @CommandLine.Option(names = {"-u", "--user"},
+            description = "Username of the user that started this job. Defaults to current user",
+            defaultValue = "${sys:user.name}")
+    private String username;
+
+    @CommandLine.Option(names = {"-e", "--email"},
+            description = "Email of the user that started this job")
+    private String emailAddress;
+
+    public String getActionName() {
+        return actionName;
+    }
+
+    public void setActionName(String actionName) {
+        this.actionName = actionName;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getEmailAddress() {
+        return emailAddress;
+    }
+
+    public void setEmailAddress(String emailAddress) {
+        this.emailAddress = emailAddress;
+    }
+}

src/test/java/edu/unc/lib/boxc/migration/cdm/jobs/VelocicroptorRemoteJobTest.java

@@ -0,0 +1,137 @@
+package edu.unc.lib.boxc.migration.cdm.jobs;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
+import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
+import edu.unc.lib.boxc.migration.cdm.options.ProcessSourceFilesOptions;
+import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory;
+import edu.unc.lib.boxc.migration.cdm.services.SourceFilesToRemoteService;
+import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
+import edu.unc.lib.boxc.migration.cdm.util.SshClientService;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.openMocks;
+
+/**
+ * @author bbpennel
+ */
+public class VelocicroptorRemoteJobTest {
+    private static final String PROJECT_NAME = "proj";
+    private static final String ADMIN_EMAIL = "[email protected]";
+    private static final String USER_EMAIL = "[email protected]";
+    private static final String USERNAME = "chompb_user";
+    private static final String OUTPUT_SERVER = "chompb.example.com";
+    private VelocicroptorRemoteJob job;
+    @Mock
+    private SshClientService sshClientService;
+    private MigrationProject project;
+    @Mock
+    private SourceFilesToRemoteService sourceFilesToRemoteService;
+    private Path remoteProjectsPath;
+    private Path remoteJobScriptPath;
+    private Path projectPath;
+    private Path outputPath;
+    @Captor
+    private ArgumentCaptor<String> remoteArgumentCaptor;
+
+    private AutoCloseable closeable;
+    @TempDir
+    public Path tmpFolder;
+
+    @BeforeEach
+    public void setup() throws IOException {
+        closeable = openMocks(this);
+        projectPath = tmpFolder.resolve(PROJECT_NAME);
+        project = MigrationProjectFactory.createCdmMigrationProject(
+                tmpFolder, PROJECT_NAME, null, USERNAME,
+                null, BxcEnvironmentHelper.DEFAULT_ENV_ID);
+        remoteProjectsPath = tmpFolder.resolve("remote_projects");
+        remoteJobScriptPath = tmpFolder.resolve("remote_job_script.sh");
+
+        outputPath = projectPath.resolve(VelocicroptorRemoteJob.RESULTS_REL_PATH);
+
+        job = new VelocicroptorRemoteJob();
+        job.setSshClientService(sshClientService);
+        job.setProject(project);
+        job.setSourceFilesToRemoteService(sourceFilesToRemoteService);
+        job.setRemoteProjectsPath(remoteProjectsPath);
+        job.setRemoteJobScriptPath(remoteJobScriptPath);
+        job.setAdminEmail(ADMIN_EMAIL);
+        job.setOutputServer(OUTPUT_SERVER);
+        job.setOutputPath(outputPath);
+    }
+
+    @AfterEach
+    void tearDown() throws Exception {
+        closeable.close();
+    }
+
+    // Successful
+    // IOException
+
+    @Test
+    public void runSuccessfulTest() throws Exception {
+        var options = new ProcessSourceFilesOptions();
+        options.setActionName("velocicroptor");
+        options.setUsername(USERNAME);
+        options.setEmailAddress(USER_EMAIL);
+
+        var jobId = job.run(options);
+
+        assertTrue(Files.isDirectory(outputPath));
+
+        var remoteProjectPath = remoteProjectsPath.resolve(project.getProjectName() + "/velocicroptor/" + jobId);
+        var sourceFilesDestinationPath = remoteProjectPath.resolve("source_files");
+        verify(sourceFilesToRemoteService).transferFiles(eq(sourceFilesDestinationPath));
+        verify(sshClientService).executeRemoteCommand(remoteArgumentCaptor.capture());
+
+        var arguments = remoteArgumentCaptor.getValue().split(" ", 3);
+        assertEquals("sbatch", arguments[0]);
+        assertEquals(remoteJobScriptPath.toString(), arguments[1]);
+
+        // Parse the config (with outer quotes trimmed off)
+        ObjectMapper mapper = new ObjectMapper();
+        var config = mapper.readTree(arguments[2].substring(1, arguments[2].length() - 1));
+        assertEquals(jobId, config.get("job_id").asText());
+        assertEquals("velocicroptor", config.get("job_name").asText());
+        assertEquals(project.getProjectName(), config.get("chompb_proj_name").asText());
+        assertEquals(ADMIN_EMAIL, config.get("admin_address").asText());
+        assertEquals(USERNAME, config.get("username").asText());
+        assertEquals(USER_EMAIL, config.get("email_address").asText());
+        assertFalse(config.get("start_time").asText().isEmpty());
+        assertEquals(outputPath.toString(), config.get("output_path").asText());
+        assertEquals(OUTPUT_SERVER, config.get("output_server").asText());
+    }
+
+    @Test
+    public void runFailsTransferFailsTest() throws Exception {
+        var options = new ProcessSourceFilesOptions();
+        options.setActionName("velocicroptor");
+        options.setUsername(USERNAME);
+        options.setEmailAddress(USER_EMAIL);
+
+        doThrow(new IOException("Failed to transfer files")).when(sourceFilesToRemoteService).transferFiles(any());
+
+        assertThrows(MigrationException.class, () -> job.run(options));
+    }
+}