From 8f9e212640702081b5e13af3d1b7ff3bd8a2aee2 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 1 Oct 2024 13:46:59 -0400 Subject: [PATCH 1/4] No Longer Use Class Globals When Exporting --- docker/swarm/README_zeke_stack_on_linux.md | 2 +- .../vcell/export/server/ExportServiceImpl.java | 18 ++++++++---------- .../cbit/vcell/export/server/N5Exporter.java | 14 +++++--------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/docker/swarm/README_zeke_stack_on_linux.md b/docker/swarm/README_zeke_stack_on_linux.md index f18809c8c5..97b7d0bf65 100644 --- a/docker/swarm/README_zeke_stack_on_linux.md +++ b/docker/swarm/README_zeke_stack_on_linux.md @@ -11,7 +11,7 @@ 2. ```bash pushd ../build - ./build.sh --skip-maven --skip-singularity --skip-sudo all localhost:5000/virtualcell dev_zeke + ./build.sh --skip-maven --skip-sudo all localhost:5000/virtualcell dev_zeke popd ``` diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java index 6d1efd8fdd..b41422a81c 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java @@ -53,12 +53,6 @@ public class ExportServiceImpl implements ExportConstants, ExportService { private Hashtable jobRequestIDs = new Hashtable(); private Hashtable completedExportRequests = new Hashtable(); - - private ASCIIExporter asciiExporter = new ASCIIExporter(this); - private IMGExporter imgExporter = new IMGExporter(this); - private RasterExporter rrExporter = new RasterExporter(this); - - private N5Exporter n5Exporter = new N5Exporter(this); /** @@ -268,6 +262,10 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer FileDataContainerManager fileDataContainerManager = new FileDataContainerManager(); try{ ExportOutput[] exportOutputs = null; + RasterExporter rrExporter = new RasterExporter(this); + IMGExporter imgExporter = new IMGExporter(this); + ASCIIExporter asciiExporter = new ASCIIExporter(this); + switch (exportSpecs.getFormat()) { case CSV: case HDF5: @@ -313,9 +311,9 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer exportOutputs = rrExporter.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case N5: - n5Exporter.initalizeDataControllers(user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier()); + N5Exporter n5Exporter = new N5Exporter(this, user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier()); ExportOutput exportOutput = n5Exporter.makeN5Data(outputContext, newExportJob, exportSpecs, fileDataContainerManager); - return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob); + return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob, n5Exporter.getN5FilePathSuffix()); default: throw new DataAccessException("Unknown export format requested"); } @@ -475,11 +473,11 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase } -private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob) throws DataFormatException, IOException{ +private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob, String pathSuffix) throws DataFormatException, IOException{ if (exportOutput.isValid()) { completedExportRequests.put(exportSpecs, newExportJob); String url = PropertyLoader.getRequiredProperty(PropertyLoader.s3ExportBaseURLProperty); - url += "/" + n5Exporter.getN5FilePathSuffix(); + url += "/" + pathSuffix; N5Specs n5Specs = (N5Specs) exportSpecs.getFormatSpecificSpecs(); url += "?dataSetName=" + newExportJob.getExportJobID(); if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + fileName); diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java index 57f6253de0..504e64599e 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java @@ -64,9 +64,9 @@ public class N5Exporter implements ExportConstants { )); - public N5Exporter(ExportServiceImpl exportServiceImpl) { - this.exportServiceImpl = exportServiceImpl; -} + public N5Exporter(ExportServiceImpl exportServiceImpl, User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) { + this.exportServiceImpl = exportServiceImpl; + } private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs n5Specs, ExportSpecs exportSpecs, FileDataContainerManager fileDataContainerManager) throws Exception { VCUnitDefinition lengthUnit = ModelUnitSystem.createDefaultVCModelUnitSystem().getLengthUnit(); @@ -139,10 +139,12 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs int timeLoops = 1; double progress = 0; + for (int variableIndex=0; variableIndex < (numVariables -1); variableIndex++){ for (int timeIndex=timeSpecs.getBeginTimeIndex(); timeIndex <= timeSpecs.getEndTimeIndex(); timeIndex++){ int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex(); + System.out.print("Writing with VCid: " + vcDataID + "."); double[] data = this.dataServer.getSimDataBlock(outputContext, user, this.vcDataID, variableNames[variableIndex], allTimes[timeIndex]).getData(); data = containsPostProcessed ? data : MeshToImage.convertMeshIntoImage(data, mesh).data(); for (int z=0; z < sizeZ; z++){ @@ -195,12 +197,6 @@ private boolean containsPostProcessedVariable(String[] variableNames, OutputCont return false; } - public void initalizeDataControllers(User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) throws IOException, DataAccessException { - this.user = user; - this.vcDataID = vcSimulationDataIdentifier; - this.dataServer = dataServer; - } - public VCSimulationDataIdentifier getVcDataID(){return vcDataID;} From cdf1df19960a08d54abf4480ee64bda17395f2ae Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 2 Oct 2024 07:19:25 -0400 Subject: [PATCH 2/4] Fix N5 Tests --- .../src/main/java/cbit/vcell/export/server/N5Exporter.java | 3 +++ .../src/test/java/cbit/vcell/export/N5ExporterTest.java | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java index 504e64599e..209f1dece7 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java @@ -66,6 +66,9 @@ public class N5Exporter implements ExportConstants { public N5Exporter(ExportServiceImpl exportServiceImpl, User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) { this.exportServiceImpl = exportServiceImpl; + this.user = user; + this.dataServer = dataServer; + this.vcDataID = vcSimulationDataIdentifier; } private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs n5Specs, ExportSpecs exportSpecs, FileDataContainerManager fileDataContainerManager) throws Exception { diff --git a/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java b/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java index ae26485fe4..4a6fee1aff 100644 --- a/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java +++ b/vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java @@ -60,6 +60,7 @@ enum TestModels{ private User testUser; private CartesianMesh modelMesh; private double[] times; + private ExportServiceImpl exportService = new ExportServiceImpl(); private ArrayList dataIdentifiers; @@ -129,8 +130,6 @@ public void setUp() throws IOException { previousSimCacheSize = PropertyLoader.getProperty(PropertyLoader.simdataCacheSizeProperty, null); PropertyLoader.setProperty(PropertyLoader.simdataCacheSizeProperty, "100000"); - ExportServiceImpl exportService = new ExportServiceImpl(); - Cachetable cachetable = new Cachetable(10 * Cachetable.minute, Long.parseLong(PropertyLoader.getRequiredProperty(PropertyLoader.simdataCacheSizeProperty))); File primaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty)); File secodaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.secondarySimDataDirInternalProperty)); @@ -138,7 +137,6 @@ public void setUp() throws IOException { DataServerImpl dataServer = new DataServerImpl(dataSetController, exportService); testUser = new User("ezequiel23", new KeyValue("258925427")); - n5Exporter = new N5Exporter(exportService); this.dataServer = dataServer; } @@ -202,7 +200,7 @@ private void setExportTestState(TestModels simModel) throws IOException, DataAcc VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(new KeyValue(simModel.simID), testUser); vcDataID = new VCSimulationDataIdentifier(vcSimulationIdentifier, 0); - n5Exporter.initalizeDataControllers(testUser, dataServer, vcDataID); + n5Exporter = new N5Exporter(exportService, testUser, dataServer, vcDataID); dataIdentifiers = new ArrayList<>(Arrays.asList(dataServer.getDataIdentifiers(new OutputContext(new AnnotatedFunction[0]), testUser, vcDataID))); modelMesh = dataServer.getMesh(testUser, vcDataID); From 8291f9bd9d0fc03cd6d4d79c894b61de88212b81 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 9 Oct 2024 10:34:14 -0400 Subject: [PATCH 3/4] Remove Print Statements --- .../main/java/cbit/vcell/export/server/ExportServiceImpl.java | 1 - .../src/main/java/cbit/vcell/export/server/N5Exporter.java | 1 - 2 files changed, 2 deletions(-) diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java index b41422a81c..112e6b1fde 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java @@ -400,7 +400,6 @@ private ExportEvent saveResultsToRemoteFile(String fileFormat, String exportBase } ZipEntry zipEntry = new ZipEntry(filename); zipOut.putNextEntry(zipEntry); - System.out.println("writing entry "+i); exportOutputs[i].writeDataToOutputStream(zipOut,fileDataContainerManager); //zipOut.write(exportOutputs[i].getData()); } else { diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java index 209f1dece7..ff1e56f0c5 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java @@ -147,7 +147,6 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs for (int timeIndex=timeSpecs.getBeginTimeIndex(); timeIndex <= timeSpecs.getEndTimeIndex(); timeIndex++){ int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex(); - System.out.print("Writing with VCid: " + vcDataID + "."); double[] data = this.dataServer.getSimDataBlock(outputContext, user, this.vcDataID, variableNames[variableIndex], allTimes[timeIndex]).getData(); data = containsPostProcessed ? data : MeshToImage.convertMeshIntoImage(data, mesh).data(); for (int z=0; z < sizeZ; z++){ From 74dc1240e8a4fe3f54ef14405dac575b6e05bac6 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 9 Oct 2024 13:15:15 -0400 Subject: [PATCH 4/4] Separate Instance for each Switch Statement --- .../export/server/ExportServiceImpl.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java index 112e6b1fde..8e15f1676d 100644 --- a/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java +++ b/vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java @@ -262,13 +262,10 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer FileDataContainerManager fileDataContainerManager = new FileDataContainerManager(); try{ ExportOutput[] exportOutputs = null; - RasterExporter rrExporter = new RasterExporter(this); - IMGExporter imgExporter = new IMGExporter(this); - ASCIIExporter asciiExporter = new ASCIIExporter(this); - switch (exportSpecs.getFormat()) { case CSV: case HDF5: + ASCIIExporter asciiExporter = new ASCIIExporter(this); Collection asciiOut = asciiExporter.makeASCIIData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); exportOutputs = asciiOut.toArray(new ExportOutput[asciiOut.size()]); if(((ASCIISpecs)exportSpecs.getFormatSpecificSpecs()).isHDF5()) { @@ -287,6 +284,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer case GIF: case FORMAT_JPEG: case ANIMATED_GIF: + IMGExporter imgExporter = new IMGExporter(this); exportOutputs = imgExporter.makeMediaData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,clientTaskStatusSupport,fileDataContainerManager); boolean bOverrideZip = exportOutputs.length == 1; if(bSaveAsZip && !bOverrideZip){ @@ -296,19 +294,24 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer } case NRRD: // case IMAGEJ: + RasterExporter rrExporter = new RasterExporter(this); NrrdInfo[] nrrdInfos = rrExporter.makeRasterData(outputContext,newExportJob, user, dataServerImpl, exportSpecs, fileDataContainerManager); return makeRemoteFile(fileFormat, exportBaseDir, exportBaseURL, nrrdInfos, exportSpecs, newExportJob, fileDataContainerManager); case UCD: - exportOutputs = rrExporter.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterUCD = new RasterExporter(this); + exportOutputs = rrExporterUCD.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case PLY: - exportOutputs = rrExporter.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterPLY = new RasterExporter(this); + exportOutputs = rrExporterPLY.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case VTK_IMAGE: - exportOutputs = rrExporter.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterVTK = new RasterExporter(this); + exportOutputs = rrExporterVTK.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case VTK_UNSTRUCT: - exportOutputs = rrExporter.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); + RasterExporter rrExporterVTKU = new RasterExporter(this); + exportOutputs = rrExporterVTKU.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager); return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager); case N5: N5Exporter n5Exporter = new N5Exporter(this, user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier());