Skip to content

Commit

Permalink
fix quotes, substitute container paths
Browse files Browse the repository at this point in the history
  • Loading branch information
jcschaff committed Sep 27, 2024
1 parent 8ebbdba commit 2929f8c
Show file tree
Hide file tree
Showing 12 changed files with 104 additions and 67 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ public String getPrefix() {
return prefix;
}
}

private record SingularityBinding(String hostPath, String containerPath) {
public String getBinding() {
return "--bind "+hostPath+":"+containerPath;
}
}

public SlurmProxy(CommandService commandService, String htcUser) {
super(commandService, htcUser);
Expand Down Expand Up @@ -478,17 +484,28 @@ SbatchSolverComponents generateScript(String jobName, ExecutableCommand.Containe
"serverid="+serverid
};

List<SingularityBinding> bindings = List.of(
new SingularityBinding(primaryDataDirExternal, "/simdata"),
new SingularityBinding(secondaryDataDirExternal, "/simdata_secondary"),
new SingularityBinding(simDataDirArchiveExternal, simDataDirArchiveInternal),
new SingularityBinding(htclogdir_external, "/htclogs"),
new SingularityBinding(slurm_tmpdir, "/solvertmp")
);

LineStringBuilder lsb = new LineStringBuilder();
LineStringBuilder singularityLSB = new LineStringBuilder();
slurmInitSingularity(singularityLSB, primaryDataDirExternal, Optional.of(secondaryDataDirExternal), htclogdir_external,
solverDockerName, Optional.of(batchDockerName),
slurmInitSingularity(singularityLSB,
solverDockerName, Optional.of(batchDockerName), bindings,
slurm_tmpdir, slurm_singularity_cachedir, slurm_singularity_pullfolder,
simDataDirArchiveExternal, simDataDirArchiveInternal,
slurm_singularity_module_name, environmentVars);

LineStringBuilder sendFailMsgLSB = new LineStringBuilder();
sendFailMsgScript(simTask, sendFailMsgLSB, jmshost_sim_external, jmsport_sim_external, jmsuser, jmspswd);


for (SingularityBinding binding : bindings) {
commandSet.translatePaths(new File(binding.hostPath), new File(binding.containerPath));
}

final boolean hasExitProcessor = commandSet.hasExitCodeCommand();
// lsb.write("run_in_container=\"singularity /path/to/data:/simdata /path/to/image/vcell-batch.img);
LineStringBuilder callExitLSB = new LineStringBuilder();
Expand All @@ -498,6 +515,7 @@ SbatchSolverComponents generateScript(String jobName, ExecutableCommand.Containe

LineStringBuilder preProcessLSB = new LineStringBuilder();
for (ExecutableCommand ec: commandSet.getExecCommands()) {
ExecutableCommand.Container commandSet2 = new ExecutableCommand.Container();
if(ec.getCommands().get(0).equals("JavaPreprocessor64")) {
execCommandScript(preProcessLSB, hasExitProcessor, ec, "${batch_container_prefix}");
}else {
Expand Down Expand Up @@ -526,7 +544,7 @@ private void callExitScript(ExecutableCommand.Container commandSet, LineStringBu
exitCmd.stripPathFromCommand();
lsb.write("callExitProcessor( ) {");
lsb.append('\t');
lsb.write("\"${batch_container_prefix} " + exitCmd.getJoinedCommands("$1")+"\"");
lsb.write("${batch_container_prefix} " + exitCmd.getJoinedCommands("$1").trim());
lsb.write("}");
lsb.newline();
lsb.newline();
Expand Down Expand Up @@ -563,7 +581,7 @@ private void sendFailMsgScript(SimulationTask simTask, LineStringBuilder lsb, St
private void execCommandScript(LineStringBuilder lsb,final boolean hasExitProcessor, ExecutableCommand ec, String container_prefix) {
ec.stripPathFromCommand();
String cmd= ec.getJoinedCommands();
lsb.write("\"" + container_prefix + " " + cmd + "\"");
lsb.write(container_prefix + " " + cmd.trim());
lsb.write("stat=$?");
lsb.append("echo returned $stat");
lsb.newline();
Expand All @@ -578,11 +596,10 @@ private void execCommandScript(LineStringBuilder lsb,final boolean hasExitProces
}


private void slurmInitSingularity(LineStringBuilder lsb, String primaryDataDirExternal,
Optional<String> secondaryDataDirExternal, String htclogdir_external,
String solverDockerName, Optional<String> batchDockerName,
private void slurmInitSingularity(LineStringBuilder lsb,
String solverDockerName, Optional<String> batchDockerName, List<SingularityBinding> bindings,
String slurm_tmpdir, String slurm_singularity_cachedir, String slurm_singularity_pullfolder,
String simDataDirArchiveExternal, String simDataDirArchiveInternal, String singularity_module_name, String[] environmentVars) {
String singularity_module_name, String[] environmentVars) {
lsb.write("TMPDIR="+slurm_tmpdir);
lsb.write("if [ ! -e $TMPDIR ]; then mkdir -p $TMPDIR ; fi");

Expand All @@ -603,11 +620,14 @@ private void slurmInitSingularity(LineStringBuilder lsb, String primaryDataDirEx
lsb.write("env");
lsb.newline();

lsb.write("container_bindings=\"--bind "+primaryDataDirExternal+":/simdata \"");
lsb.write("container_bindings+=\"--bind "+slurm_tmpdir+":/solvertmp \"");
lsb.write("container_bindings+=\"--bind "+htclogdir_external+":/htclogs \"");
if (secondaryDataDirExternal.isPresent()) {
lsb.write("container_bindings+=\"--bind "+secondaryDataDirExternal.get()+":/simdata_secondary \"");
boolean bFirstBind=true;
for (SingularityBinding binding : bindings) {
if (bFirstBind) {
lsb.write("container_bindings=\"--bind " + binding.hostPath + ":" + binding.containerPath + " \"");
bFirstBind = false;
}else{
lsb.write("container_bindings+=\"--bind " + binding.hostPath + ":" + binding.containerPath + " \"");
}
}
boolean bFirstEnv=true;
for (String envVar : environmentVars) {
Expand Down Expand Up @@ -761,10 +781,16 @@ String createOptJobScript(String jobName, File optProblemInputFile, File optProb
if (!optDataDirExternal.exists() && !optDataDirExternal.mkdir()){
LG.error("failed to make optimization data directory "+optDataDir.getAbsolutePath());
}
// if (optDataDirExternal.setWritable(true,false))
slurmInitSingularity(lsb, optDataDirExternal.getAbsolutePath(), Optional.empty(), htclogdir_external, solverDockerName, batchDockerName,
slurm_tmpdir, slurm_singularity_cachedir, slurm_singularity_pullfolder,
simDataDirArchiveExternal, simDataDirArchiveInternal,

List<SingularityBinding> bindings = List.of(
new SingularityBinding(optDataDirExternal.getAbsolutePath(), "/simdata"),
new SingularityBinding(slurm_tmpdir, "/solvertmp"),
new SingularityBinding(htclogdir_external, "/htclogs"),
new SingularityBinding(simDataDirArchiveExternal, simDataDirArchiveInternal)
);

slurmInitSingularity(lsb, solverDockerName, batchDockerName,
bindings, slurm_tmpdir, slurm_singularity_cachedir, slurm_singularity_pullfolder,
slurm_singularity_module_name, environmentVars);

lsb.write("cmd_prefix=\"$solver_container_prefix\"");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274633859 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274633859_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274633859 schaff 17 0 0 $1 /htclogs/V_REL_274633859_0_0.slurm.sub
}


"${solver_container_prefix} JavaSimExe64 /share/apps/vcell3/users/schaff/SimID_274633859_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${solver_container_prefix} JavaSimExe64 /simdata/schaff/SimID_274633859_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274630682 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274630682_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274630682 schaff 17 0 0 $1 /htclogs/V_REL_274630682_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274630682_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274630682_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} SundialsSolverStandalone_x64 /share/apps/vcell3/users/schaff/SimID_274630682_0_.cvodeInput /share/apps/vcell3/users/schaff/SimID_274630682_0_.ida -tid 0 "
${solver_container_prefix} SundialsSolverStandalone_x64 /simdata/schaff/SimID_274630682_0_.cvodeInput /simdata/schaff/SimID_274630682_0_.ida -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274514696 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274514696_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274514696 schaff 17 0 0 $1 /htclogs/V_REL_274514696_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274514696_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274514696_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} FiniteVolume_x64 /share/apps/vcell3/users/schaff/SimID_274514696_0_.fvinput -tid 0 "
${solver_container_prefix} FiniteVolume_x64 /simdata/schaff/SimID_274514696_0_.fvinput -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274635122 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274635122_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274635122 schaff 17 0 0 $1 /htclogs/V_REL_274635122_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274635122_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274635122_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} VCellStoch_x64 gibson /share/apps/vcell3/users/schaff/SimID_274635122_0_.stochInput /share/apps/vcell3/users/schaff/SimID_274635122_0_.ida -tid 0 "
${solver_container_prefix} VCellStoch_x64 gibson /simdata/schaff/SimID_274635122_0_.stochInput /simdata/schaff/SimID_274635122_0_.ida -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274641698 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274641698_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274641698 schaff 17 0 0 $1 /htclogs/V_REL_274641698_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274641698_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274641698_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} Hybrid_EM_x64 /share/apps/vcell3/users/schaff/SimID_274641698_0_.nc 100.0 10.0 0.01 0.1 -OV -tid 0 "
${solver_container_prefix} Hybrid_EM_x64 /simdata/schaff/SimID_274641698_0_.nc 100.0 10.0 0.01 0.1 -OV -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274672135 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274672135_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274672135 schaff 17 0 0 $1 /htclogs/V_REL_274672135_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274672135_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274672135_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} langevin_x64 simulate --output-log=/share/apps/vcell3/users/schaff/SimID_274672135_0_.log --vc-send-status-config=/share/apps/vcell3/users/schaff/SimID_274672135_0_.langevinMessagingConfig /share/apps/vcell3/users/schaff/SimID_274672135_0_.langevinInput 0 -tid 0 "
${solver_container_prefix} langevin_x64 simulate --output-log=/simdata/schaff/SimID_274672135_0_.log --vc-send-status-config=/simdata/schaff/SimID_274672135_0_.langevinMessagingConfig /simdata/schaff/SimID_274672135_0_.langevinInput 0 -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ echo ENVIRONMENT
env

container_bindings="--bind /share/apps/vcell3/users:/simdata "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /share/apps/vcell7/users:/simdata_secondary "
container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users "
container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
container_bindings+="--bind /scratch/vcell:/solvertmp "
container_env="--env java_mem_Xmx=4096M "
container_env+="--env jmshost_sim_internal=rke-wn-01.cam.uchc.edu "
container_env+="--env jmsport_sim_internal=31618 "
Expand Down Expand Up @@ -61,11 +62,11 @@ sendFailureMsg() {


callExitProcessor( ) {
"${batch_container_prefix} JavaPostprocessor64 274641196 schaff 17 0 0 $1 /share/apps/vcell3/htclogs/V_REL_274641196_0_0.slurm.sub "
${batch_container_prefix} JavaPostprocessor64 274641196 schaff 17 0 0 $1 /htclogs/V_REL_274641196_0_0.slurm.sub
}


"${batch_container_prefix} JavaPreprocessor64 /share/apps/vcell3/users/schaff/SimID_274641196_0__0.simtask.xml /share/apps/vcell3/users/schaff "
${batch_container_prefix} JavaPreprocessor64 /simdata/schaff/SimID_274641196_0__0.simtask.xml /simdata/schaff
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand All @@ -74,7 +75,7 @@ if [ $stat -ne 0 ]; then
exit $stat
fi

"${solver_container_prefix} MovingBoundary_x64 --config /share/apps/vcell3/users/schaff/SimID_274641196_0_mb.xml -tid 0 "
${solver_container_prefix} MovingBoundary_x64 --config /simdata/schaff/SimID_274641196_0_mb.xml -tid 0
stat=$?
echo returned $stat
if [ $stat -ne 0 ]; then
Expand Down
Loading

0 comments on commit 2929f8c

Please sign in to comment.