Merge pull request #1359 from virtualcell/use-vcell-fvsolver

simplified Slurm scripts, uses vcell-solvers and vcell-fvsolver containers
virtualcell · Sep 27, 2024 · 9156f37 · 9156f37
2 parents 0995c9e + f096458
commit 9156f37
Show file tree

Hide file tree

Showing 20 changed files with 601 additions and 1,669 deletions.
diff --git a/.github/workflows/CI-full.yml b/.github/workflows/CI-full.yml
@@ -94,34 +94,22 @@ jobs:
         java-version: '17'
         cache: 'maven'
 
-    - name: Install Singularity # to make singularity image for cluster
-      uses: eWaterCycle/setup-singularity@v6
-      with:
-        singularity-version: 3.7.1
     - name: build and publish all images
       shell: bash
       run: |
         cd docker/build
         sudo docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
         docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
         ./build.sh all ${{ env.VCELL_REPO_NAMESPACE }} ${{ env.VCELL_TAG }}
-        cd singularity-vm
-        singularity remote login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} oras://ghcr.io
-    - name: tag as latest and push to registry  # (jcs) are explicit singularity push commands redundant? (see ./build.sh)
+
+    - name: tag as latest and push to registry
       shell: bash
       run: |
         for CONTAINER in vcell-api vcell-rest vcell-webapp-prod vcell-webapp-dev vcell-webapp-stage vcell-webapp-island vcell-batch vcell-opt vcell-clientgen vcell-data vcell-db vcell-mongo vcell-sched vcell-submit vcell-admin;\
-        do docker tag ${VCELL_REPO_NAMESPACE}/$CONTAINER:${VCELL_TAG} ${VCELL_REPO_NAMESPACE}/$CONTAINER:latest;\
-        docker tag ${VCELL_REPO_NAMESPACE}/$CONTAINER:${VCELL_TAG} ${VCELL_REPO_NAMESPACE}/$CONTAINER:${{ steps.version.outputs.tag }};\
-        docker push --all-tags ${VCELL_REPO_NAMESPACE}/$CONTAINER;\
+           do docker tag ${VCELL_REPO_NAMESPACE}/$CONTAINER:${VCELL_TAG} ${VCELL_REPO_NAMESPACE}/$CONTAINER:latest;\
+           docker tag ${VCELL_REPO_NAMESPACE}/$CONTAINER:${VCELL_TAG} ${VCELL_REPO_NAMESPACE}/$CONTAINER:${{ steps.version.outputs.tag }};\
+           docker push --all-tags ${VCELL_REPO_NAMESPACE}/$CONTAINER;\
         done
-        cd docker/build/singularity-vm
-        singularity push -U $(ls *batch*img) oras://${VCELL_REPO_NAMESPACE}/vcell-batch-singularity:${VCELL_TAG}
-        singularity push -U $(ls *batch*img) oras://${VCELL_REPO_NAMESPACE}/vcell-batch-singularity:${{ steps.version.outputs.tag }}
-        singularity push -U $(ls *batch*img) oras://${VCELL_REPO_NAMESPACE}/vcell-batch-singularity:latest
-        singularity push -U $(ls *opt*img) oras://${VCELL_REPO_NAMESPACE}/vcell-opt-singularity:${VCELL_TAG}
-        singularity push -U $(ls *opt*img) oras://${VCELL_REPO_NAMESPACE}/vcell-opt-singularity:${{ steps.version.outputs.tag }}
-        singularity push -U $(ls *opt*img) oras://${VCELL_REPO_NAMESPACE}/vcell-opt-singularity:latest
 
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3

diff --git a/.github/workflows/site_deploy.yml b/.github/workflows/site_deploy.yml
@@ -182,20 +182,6 @@ jobs:
         ssh-keyscan $VCELL_MANAGER_NODE >> ~/.ssh/known_hosts
         cd docker/swarm
         scp ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE}:${VCELL_DEPLOY_REMOTE_DIR}/${VCELL_CONFIG_FILE_NAME} .
-    - name: install singularity
-      uses: eWaterCycle/setup-singularity@v6
-      with:
-        singularity-version: 3.7.1
-    - name: retrieve batch and opt singularity images
-      run: |
-        set -ux
-        cd docker/swarm
-        export BATCH_SINGULARITY_FILENAME=`cat $VCELL_CONFIG_FILE_NAME | grep VCELL_BATCH_SINGULARITY_FILENAME | cut -d"=" -f2`
-        export OPT_SINGULARITY_FILENAME=`cat $VCELL_CONFIG_FILE_NAME | grep VCELL_OPT_SINGULARITY_FILENAME | cut -d"=" -f2`
-        cd ../build/singularity-vm
-        singularity remote login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} oras://ghcr.io
-        singularity pull $BATCH_SINGULARITY_FILENAME oras://${VCELL_REPO_NAMESPACE}/vcell-batch-singularity:${{ github.event.inputs.vcell_version }}.${{ github.event.inputs.vcell_build }}
-        singularity pull $OPT_SINGULARITY_FILENAME   oras://${VCELL_REPO_NAMESPACE}/vcell-opt-singularity:${{ github.event.inputs.vcell_version }}.${{ github.event.inputs.vcell_build }}
     - name: setup java 17 with maven cache (for documentation build)
       uses: actions/setup-java@v4
       with:
@@ -207,16 +193,15 @@ jobs:
       run: |
         set -ux
         mvn clean install -DskipTests
-    - name: deploy installers and singularity to kubernetes site and web help to vcell.org
+    - name: deploy installers and web help to vcell.org
       run: |
         set -ux
         cd docker/swarm
         ssh -t ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE} sudo docker login -u ${{ secrets.ACTION_USER }} -p ${{ secrets.ACTION_TOKEN }} ghcr.io
         if ${{ github.event.inputs.server_only != 'true' }}; then
-          # build and install the client installers, the singularity images, and the web help (kubernetes cluster deployments are separate)
+          # build and install the client installers, and the web help (kubernetes cluster deployments are separate)
           ./deploy-action-kubernetes.sh \
               --ssh-user ${{ secrets.CD_FULL_USER }} \
-              --install-singularity \
               --build-installers \
               --installer-deploy-dir $VCELL_INSTALLER_REMOTE_DIR \
               --webhelp-local-dir ../../vcell-client/target/classes/vcellDoc \
@@ -227,13 +212,6 @@ jobs:
           ssh ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE} \
             installer_deploy_dir=$VCELL_INSTALLER_REMOTE_DIR vcell_siteCamel=$VCELL_SITE_CAMEL vcell_version=$VCELL_VERSION vcell_build=$VCELL_BUILD \
             'bash -s' < link-installers.sh
-        else
-          # build and install only the singularity images (kubernetes cluster deployments are separate)
-          ./deploy-action-kubernetes.sh \
-              --ssh-user ${{ secrets.CD_FULL_USER }} \
-              --install-singularity  \
-              ${VCELL_MANAGER_NODE} \ 
-              ./${VCELL_CONFIG_FILE_NAME}
         fi
     - name: Capitalize first character of site name
       id: capitalize

diff --git a/docker/build/Dockerfile-submit-dev b/docker/build/Dockerfile-submit-dev
@@ -64,9 +64,13 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
     htclogdir_external=/path/to/external/htclogs/ \
     nativesolverdir_external=/path/to/external/nativesolvers/ \
     htcnodelist="batch-host-not-set" \
-    batch_singularity_imagefile=/path/to/external/batch/singularity.img \
-    opt_singularity_imagefile=/path/to/external/opt/singularity_opt.img \
-    docker_name="repo/namespace/vcell-batch:tag" \
+    htc_vcellfvsolver_docker_name="htc-vcellfvsolver-docker-name-not-set" \
+    htc_vcellfvsolver_solver_list="htc-vcellfvsolver-solver-list-not-set" \
+    htc_vcellsolvers_docker_name="htc-vcellsolvers-docker-name-not-set" \
+    htc_vcellsolvers_solver_list="htc-vcellsolvers-solver-list-not-set" \
+    htc_vcellbatch_docker_name="htc-vcellbatch-docker-name-not-set" \
+    htc_vcellbatch_solver_list="htc-vcellbatch-solver-list-not-set" \
+    htc_vcellopt_docker_name="htc-vcellopt-docker-name-not-set" \
     batchhost="batch-host-not-set" \
     batchuser="batch-user-not-set" \
     slurm_cmd_sbatch=sbatch \
@@ -80,8 +84,8 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \
     slurm_reservation_pu="slurm_reservation_pu-not-set" \
     slurm_qos_pu="slurm_qos_pu-not-set" \
     slurm_tmpdir="slurm-tmpdir-not-set" \
-    slurm_local_singularity_dir="slurm_local_singularity_dir-not-set" \
-    slurm_central_singularity_dir="slurm_central_singularity_dir-not-set" \
+    slurm_singularity_cachedir="slurm_singularity_cachedir-not-set" \
+    slurm_singularity_pullfolder="slurm_singularity_pullfolder-not-set" \
     slurm_singularity_module_name="slurm_singularity_module_name-not-set" \
     jmsblob_minsize=100000 \
     vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \
@@ -131,12 +135,16 @@ ENTRYPOINT java \
 	-Dvcell.slurm.reservationpu="${slurm_reservation_pu}" \
 	-Dvcell.slurm.qospu="${slurm_qos_pu}" \
 	-Dvcell.slurm.tmpdir="${slurm_tmpdir}" \
-	-Dvcell.slurm.local.singularity.dir="${slurm_local_singularity_dir}" \
-	-Dvcell.slurm.central.singularity.dir="${slurm_central_singularity_dir}" \
+	-Dvcell.slurm.singularity.cachedir="${slurm_singularity_cachedir}" \
+	-Dvcell.slurm.singularity.pullfolder="${slurm_singularity_pullfolder}" \
 	-Dvcell.slurm.singularity.module.name="${slurm_singularity_module_name}" \
-	-Dvcell.batch.singularity.image="${batch_singularity_imagefile}" \
-	-Dvcell.opt.singularity.image="${opt_singularity_imagefile}" \
-	-Dvcell.batch.docker.name="${docker_name}" \
+	-Dvcell.htc.vcellfvsolver.docker.name="${htc_vcellfvsolver_docker_name}" \
+	-Dvcell.htc.vcellfvsolver.solver.list="${htc_vcellfvsolver_solver_list}" \
+	-Dvcell.htc.vcellsolvers.docker.name="${htc_vcellsolvers_docker_name}" \
+	-Dvcell.htc.vcellsolvers.solver.list="${htc_vcellsolvers_solver_list}" \
+	-Dvcell.htc.vcellbatch.docker.name="${htc_vcellbatch_docker_name}" \
+	-Dvcell.htc.vcellbatch.solver.list="${htc_vcellbatch_solver_list}" \
+	-Dvcell.htc.vcellopt.docker.name="${htc_vcellopt_docker_name}" \
 	-Dvcell.simulation.postprocessor=JavaPostprocessor64 \
 	-Dvcell.simulation.preprocessor=JavaPreprocessor64 \
 	-Dvcell.javaSimulation.executable=JavaSimExe64 \

diff --git a/docker/build/build.sh b/docker/build/build.sh
@@ -6,7 +6,6 @@ ssh_user=$(whoami)
 ssh_key=
 skip_push=false
 skip_maven=false
-skip_singularity=false
 SUDO_CMD=sudo
 mvn_repo=$HOME/.m2
 
@@ -29,8 +28,6 @@ show_help() {
 	echo ""
 	echo "    --ssh-key  keyfile    ssh key for passwordless ssh to node"
 	echo ""
-	echo "    --skip-singularity    skip build of Singularity image for vcell-batch and vcell-opt containers (stored in ./singularity/)"
-	echo ""
 	echo "    --skip-maven          skip vcell software build prior to building containers"
 	echo ""
 	echo "    --skip-push           skip pushing containers to repository"
@@ -72,9 +69,6 @@ while :; do
 		--skip-sudo)
 			SUDO_CMD=
 			;;
-		--skip-singularity)
-			skip_singularity=true
-			;;
 		-?*)
 			printf 'ERROR: Unknown option: %s\n' "$1" >&2
 			echo ""
@@ -229,127 +223,6 @@ build_mongo() {
 	fi
 }
 
-build_batch_singularity() {
-  # turn on logging of commands
-  set -x
-	if [ "$skip_singularity" == "false" ]; then
-		if [ -x "$(command -v singularity)" ]; then
-			build_batch_singularity_direct
-			if [[ $? -ne 0 ]]; then echo "failed to build singularity image using singularity commands"; exit 1; fi
-		else 
-      echo "singularity not found, cannot build singularity image"
-      exit 1
-    fi
-	fi
-	# turn off logging of commands
-	set +x
-}
-
-build_batch_singularity_direct() {
-
-	echo ""
-	cmd="cd singularity-vm"
-	cd singularity-vm
-	echo ""
-	echo "CURRENT DIRECTORY IS $PWD"
-
-	#
-	# create temporary Singularity file which imports existing docker image from registry and adds a custom entrypoint
-	#
-	_vcell_batch_docker_name="${repo}/vcell-batch:${tag}"
-	_singularity_image_file="${_vcell_batch_docker_name//[\/:]/_}.img"
-	_singularity_file="Singularity_${_vcell_batch_docker_name//[\/:]/_}"
-
-cat <<EOF >$_singularity_file
-Bootstrap: docker
-From: $_vcell_batch_docker_name
-
-%runscript
-
-    exec /vcellscripts/entrypoint.sh "\$@"
-
-%labels
-
-AUTHOR jcschaff
-EOF
-
-	echo ""
-	echo "wrote Singularity file $_singularity_file"
-	cat $_singularity_file
-
-	#
-	# build the singularity image and place in singularity-vm directory
-	#
-	echo ""
-	remote_cmd1="sudo singularity build $_singularity_image_file $_singularity_file"
-	remote_cmd2="singularity build --fakeroot $_singularity_image_file $_singularity_file"
-	echo "$remote_cmd1"
-	($remote_cmd1)
-	if [[ $? -ne 0 ]]
-	then
-		echo "failed to build singularity image with sudo, will try fakeroot"
-		echo "$remote_cmd2"
-		($remote_cmd2)
-		if [[ $? -ne 0 ]]; then echo "failed to build singularity image with fakeroot"; exit 1; fi		
-	fi
-
-	echo ""
-	echo "created Singularity image for vcell-bash ./$_singularity_image_file locally (in ./singularity-vm folder), can be pushed to remote server during deploy"
-	echo ""
-	echo "cd .."
-	cd ..
-}
-
-
-build_opt_singularity() {
-	if [ "$skip_singularity" == "false" ]; then
-		if [ -x "$(command -v singularity)" ]; then
-			build_opt_singularity_direct
-			if [[ $? -ne 0 ]]; then echo "failed to build opt singularity image using singularity commands"; exit 1; fi
-		else
-      echo "singularity not found, cannot build opt singularity image"
-      exit 1
-    fi
-	fi
-}
-
-build_opt_singularity_direct() {
-
-	echo ""
-	cmd="cd singularity-vm"
-	cd singularity-vm
-	echo ""
-	echo "CURRENT DIRECTORY IS $PWD"
-
-	#
-	# create temporary Singularity file which imports existing docker image from registry and adds a custom entrypoint
-	#
-	_vcell_opt_docker_name="${repo}/vcell-opt:${tag}"
-	_singularity_image_file="${_vcell_opt_docker_name//[\/:]/_}.img"
-	_singularity_file="Singularity_${_vcell_opt_docker_name//[\/:]/_}"
-
-	#
-	# build the singularity image and place in singularity-vm directory
-	#
-	echo ""
-	remote_cmd1="sudo singularity build ${_singularity_image_file} docker://${_vcell_opt_docker_name}"
-	remote_cmd2="singularity build --fakeroot ${_singularity_image_file} docker://${_vcell_opt_docker_name}"
-	echo "$remote_cmd1"
-	($remote_cmd1)
-	if [[ $? -ne 0 ]]
-	then
-		echo "failed to build opt singularity image with sudo, will try fakeroot"
-		echo "$remote_cmd2"
-		($remote_cmd2)
-		if [[ $? -ne 0 ]]; then echo "failed to build opt singularity image with fakeroot"; exit 1; fi
-	fi
-
-	echo ""
-	echo "created Singularity image for vcell-opt ./$_singularity_image_file locally (in ./singularity-vm folder), can be pushed to remote server during deploy"
-	echo ""
-	echo "cd .."
-	cd ..
-}
 
 
 shift
@@ -362,11 +235,11 @@ fi
 
 case $target in
 	batch)
-		build_batch && build_batch_singularity
+		build_batch
 		exit $?
 		;;
 	opt)
-		build_opt && build_opt_singularity
+		build_opt
 		exit $?
 		;;
 	api)
@@ -414,8 +287,8 @@ case $target in
 		exit $?
 		;;
 	all)
-#		build_api && build_rest && build_db && build_sched && build_submit && build_data && build_mongo && build_batch && build_opt && build_clientgen && build_batch_singularity && build_opt_singularity && build_admin
-		build_api && build_rest && build_webapp && build_db && build_sched && build_submit && build_data && build_mongo && build_batch && build_opt && build_clientgen && build_batch_singularity && build_opt_singularity && build_admin
+#		build_api && build_rest && build_db && build_sched && build_submit && build_data && build_mongo && build_batch && build_opt && build_clientgen && build_admin
+		build_api && build_rest && build_webapp && build_db && build_sched && build_submit && build_data && build_mongo && build_batch && build_opt && build_clientgen && build_admin
 		exit $?
 		;;
 	appservices)