Skip to content

Commit

Permalink
Merge pull request #1364 from virtualcell/exporting-n5-discrepancies
Browse files Browse the repository at this point in the history 
Export Service Implementation Holding Old State
  • Loading branch information
@AvocadoMoon
AvocadoMoon authored Oct 9, 2024
2 parents e090ed3 + 74dc124 commit 09fcba6
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 29 deletions.
2 changes: 1 addition & 1 deletion docker/swarm/README_zeke_stack_on_linux.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
2.
```bash
pushd ../build
./build.sh --skip-maven --skip-singularity --skip-sudo all localhost:5000/virtualcell dev_zeke
./build.sh --skip-maven --skip-sudo all localhost:5000/virtualcell dev_zeke
popd
```
Expand Down
30 changes: 15 additions & 15 deletions vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,6 @@ public class ExportServiceImpl implements ExportConstants, ExportService {

private Hashtable<Long, User> jobRequestIDs = new Hashtable<Long, User>();
private Hashtable<ExportSpecs, JobRequest> completedExportRequests = new Hashtable<ExportSpecs, JobRequest>();

private ASCIIExporter asciiExporter = new ASCIIExporter(this);
private IMGExporter imgExporter = new IMGExporter(this);
private RasterExporter rrExporter = new RasterExporter(this);

private N5Exporter n5Exporter = new N5Exporter(this);


/**
Expand Down Expand Up @@ -271,6 +265,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
switch (exportSpecs.getFormat()) {
case CSV:
case HDF5:
ASCIIExporter asciiExporter = new ASCIIExporter(this);
Collection<ExportOutput> asciiOut = asciiExporter.makeASCIIData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
exportOutputs = asciiOut.toArray(new ExportOutput[asciiOut.size()]);
if(((ASCIISpecs)exportSpecs.getFormatSpecificSpecs()).isHDF5()) {
Expand All @@ -289,6 +284,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
case GIF:
case FORMAT_JPEG:
case ANIMATED_GIF:
IMGExporter imgExporter = new IMGExporter(this);
exportOutputs = imgExporter.makeMediaData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,clientTaskStatusSupport,fileDataContainerManager);
boolean bOverrideZip = exportOutputs.length == 1;
if(bSaveAsZip && !bOverrideZip){
Expand All @@ -298,24 +294,29 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
}
case NRRD:
// case IMAGEJ:
RasterExporter rrExporter = new RasterExporter(this);
NrrdInfo[] nrrdInfos = rrExporter.makeRasterData(outputContext,newExportJob, user, dataServerImpl, exportSpecs, fileDataContainerManager);
return makeRemoteFile(fileFormat, exportBaseDir, exportBaseURL, nrrdInfos, exportSpecs, newExportJob, fileDataContainerManager);
case UCD:
exportOutputs = rrExporter.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
RasterExporter rrExporterUCD = new RasterExporter(this);
exportOutputs = rrExporterUCD.makeUCDData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager);
case PLY:
exportOutputs = rrExporter.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
RasterExporter rrExporterPLY = new RasterExporter(this);
exportOutputs = rrExporterPLY.makePLYWithTexData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager);
case VTK_IMAGE:
exportOutputs = rrExporter.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
RasterExporter rrExporterVTK = new RasterExporter(this);
exportOutputs = rrExporterVTK.makeVTKImageData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager);
case VTK_UNSTRUCT:
exportOutputs = rrExporter.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
RasterExporter rrExporterVTKU = new RasterExporter(this);
exportOutputs = rrExporterVTKU.makeVTKUnstructuredData(outputContext,newExportJob, user, dataServerImpl, exportSpecs,fileDataContainerManager);
return saveResultsToRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager);
case N5:
n5Exporter.initalizeDataControllers(user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier());
N5Exporter n5Exporter = new N5Exporter(this, user, dataServerImpl, (VCSimulationDataIdentifier) exportSpecs.getVCDataIdentifier());
ExportOutput exportOutput = n5Exporter.makeN5Data(outputContext, newExportJob, exportSpecs, fileDataContainerManager);
return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob);
return makeRemoteN5File(fileFormat, n5Exporter.getN5FileNameHash(), exportOutput, exportSpecs, newExportJob, n5Exporter.getN5FilePathSuffix());
default:
throw new DataAccessException("Unknown export format requested");
}
Expand Down Expand Up @@ -402,7 +403,6 @@ private ExportEvent saveResultsToRemoteFile(String fileFormat, String exportBase
}
ZipEntry zipEntry = new ZipEntry(filename);
zipOut.putNextEntry(zipEntry);
System.out.println("writing entry "+i);
exportOutputs[i].writeDataToOutputStream(zipOut,fileDataContainerManager);
//zipOut.write(exportOutputs[i].getData());
} else {
Expand Down Expand Up @@ -475,11 +475,11 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase
}


private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob) throws DataFormatException, IOException{
private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportOutput exportOutput, ExportSpecs exportSpecs, JobRequest newExportJob, String pathSuffix) throws DataFormatException, IOException{
if (exportOutput.isValid()) {
completedExportRequests.put(exportSpecs, newExportJob);
String url = PropertyLoader.getRequiredProperty(PropertyLoader.s3ExportBaseURLProperty);
url += "/" + n5Exporter.getN5FilePathSuffix();
url += "/" + pathSuffix;
N5Specs n5Specs = (N5Specs) exportSpecs.getFormatSpecificSpecs();
url += "?dataSetName=" + newExportJob.getExportJobID();
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + fileName);
Expand Down
16 changes: 7 additions & 9 deletions vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,12 @@ public class N5Exporter implements ExportConstants {
));


public N5Exporter(ExportServiceImpl exportServiceImpl) {
this.exportServiceImpl = exportServiceImpl;
}
public N5Exporter(ExportServiceImpl exportServiceImpl, User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) {
this.exportServiceImpl = exportServiceImpl;
this.user = user;
this.dataServer = dataServer;
this.vcDataID = vcSimulationDataIdentifier;
}

private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs n5Specs, ExportSpecs exportSpecs, FileDataContainerManager fileDataContainerManager) throws Exception {
VCUnitDefinition lengthUnit = ModelUnitSystem.createDefaultVCModelUnitSystem().getLengthUnit();
Expand Down Expand Up @@ -139,6 +142,7 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs

int timeLoops = 1;
double progress = 0;

for (int variableIndex=0; variableIndex < (numVariables -1); variableIndex++){
for (int timeIndex=timeSpecs.getBeginTimeIndex(); timeIndex <= timeSpecs.getEndTimeIndex(); timeIndex++){
int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex();
Expand Down Expand Up @@ -195,12 +199,6 @@ private boolean containsPostProcessedVariable(String[] variableNames, OutputCont
return false;
}

public void initalizeDataControllers(User user, DataServerImpl dataServer, VCSimulationDataIdentifier vcSimulationDataIdentifier) throws IOException, DataAccessException {
this.user = user;
this.vcDataID = vcSimulationDataIdentifier;
this.dataServer = dataServer;
}


public VCSimulationDataIdentifier getVcDataID(){return vcDataID;}

Expand Down
6 changes: 2 additions & 4 deletions vcell-core/src/test/java/cbit/vcell/export/N5ExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ enum TestModels{
private User testUser;
private CartesianMesh modelMesh;
private double[] times;
private ExportServiceImpl exportService = new ExportServiceImpl();


private ArrayList<DataIdentifier> dataIdentifiers;
Expand Down Expand Up @@ -129,16 +130,13 @@ public void setUp() throws IOException {
previousSimCacheSize = PropertyLoader.getProperty(PropertyLoader.simdataCacheSizeProperty, null);
PropertyLoader.setProperty(PropertyLoader.simdataCacheSizeProperty, "100000");

ExportServiceImpl exportService = new ExportServiceImpl();

Cachetable cachetable = new Cachetable(10 * Cachetable.minute, Long.parseLong(PropertyLoader.getRequiredProperty(PropertyLoader.simdataCacheSizeProperty)));
File primaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirInternalProperty));
File secodaryDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.secondarySimDataDirInternalProperty));
DataSetControllerImpl dataSetController = new DataSetControllerImpl(cachetable, primaryDir, secodaryDir);
DataServerImpl dataServer = new DataServerImpl(dataSetController, exportService);

testUser = new User("ezequiel23", new KeyValue("258925427"));
n5Exporter = new N5Exporter(exportService);
this.dataServer = dataServer;
}

Expand Down Expand Up @@ -202,7 +200,7 @@ private void setExportTestState(TestModels simModel) throws IOException, DataAcc
VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(new KeyValue(simModel.simID), testUser);

vcDataID = new VCSimulationDataIdentifier(vcSimulationIdentifier, 0);
n5Exporter.initalizeDataControllers(testUser, dataServer, vcDataID);
n5Exporter = new N5Exporter(exportService, testUser, dataServer, vcDataID);
dataIdentifiers = new ArrayList<>(Arrays.asList(dataServer.getDataIdentifiers(new OutputContext(new AnnotatedFunction[0]), testUser, vcDataID)));

modelMesh = dataServer.getMesh(testUser, vcDataID);
Expand Down

0 comments on commit 09fcba6

Please sign in to comment.