Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

added --init-commands option to put a line into job script #17

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 36 additions & 2 deletions galyleo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
# future, may include a Makefile with a PREFIX option to install
# correctly and set this variable. e.g., https://github.com/oyvinev/log.sh

declare -xr GALYLEO_INSTALL_DIR="${PWD}"
declare -xr GALYLEO_INSTALL_DIR="/home/kenneth/info/elba/galyleo"

# Declare a global environment variable to set the galyleo cache
# directory, which will hold all output files generated by galyleo.
Expand Down Expand Up @@ -106,6 +106,7 @@ fi
# -s | --sif <singularity_image_file>
# -B | --bind <singularity_bind_mounts>
# | --nv
# | --init-commands <init_commands>
# | --conda-init <conda_init>
# | --conda-env <conda_env>
# | --conda-yml <conda_yml>
Expand Down Expand Up @@ -152,6 +153,9 @@ function galyleo_launch() {
# Declare input variables associated with environment modules.
local env_modules=''

# Declare input variables associated with init script.
local init_commands=''

# Declare input variables associated with Singularity containers.
local singularity_image_file=''
local singularity_bind_mounts=''
Expand Down Expand Up @@ -263,6 +267,10 @@ function galyleo_launch() {
singularity_gpu_type='rocm'
shift 1
;;
--init-commands )
init_commands="${2}"
shift 2
;;
--conda-init )
conda_init="${2}"
shift 2
Expand Down Expand Up @@ -326,6 +334,7 @@ function galyleo_launch() {
slog output -m " -s | --sif : ${singularity_image_file}"
slog output -m " -B | --bind : ${singularity_bind_mounts}"
slog output -m " | --nv : ${singularity_gpu_type}"
slog output -m " | --init-commands : ${init_commands}"
slog output -m " | --conda-init : ${conda_init}"
slog output -m " | --conda-env : ${conda_env}"
slog output -m " | --conda-yml : ${conda_yml}"
Expand Down Expand Up @@ -398,6 +407,16 @@ function galyleo_launch() {
done
fi

# Check if init line specified by the user, if any, is
# available and can be run successfully. If not, then halt the launch.
if [[ -n "${init_commands}" ]]; then
sh -c "${init_commands}"
if [[ $? -ne 0 ]]; then
slog error -m "init-commands failed: ${init_commands}"
return 1
fi
fi

# Check if the conda environment specified by the user, if any, can be
# initialized and activated successfully. If not, then halt the launch.
if [[ -n "${conda_env}" ]] && [[ -z "${conda_yml}" ]]; then
Expand Down Expand Up @@ -446,7 +465,8 @@ function galyleo_launch() {
fi
else
if [[ -z "${conda_yml}" ]]; then
jupyter "${jupyter_interface}" --version > /dev/null
#jupyter "${jupyter_interface}" --version > /dev/null
which jupyter
if [[ "${?}" -ne 0 ]]; then
slog error -m "No jupyter executable was found within the software environment."
return 1
Expand All @@ -468,15 +488,18 @@ function galyleo_launch() {
slog error -m "Unable to connect to the Satellite reverse proxy service: ${http_status_code}"
return 1
fi
#: # don't run, just testing

# Extract the reverse proxy connection token and export it as a
# read-only environment variable.
declare -xr REVERSE_PROXY_TOKEN="$(echo ${http_response} | awk 'NF>1{printf $((NF-1))}' -)"
#: # don't run, just testing

# Generate an authentication token to be used for first-time
# connections to the Jupyter notebook server and export it as a
# read-only environment variable.
declare -xr JUPYTER_TOKEN="$(openssl rand -hex 16)"
#: # don't run, just testing

# Change present working directory to GALYLEO_CACHE_DIR. Generate and
# store all Jupyter launch scripts and standard output files in the
Expand All @@ -499,6 +522,7 @@ function galyleo_launch() {

# Generate a Jupyter launch script.
slog output -m 'Generating Jupyter launch script ...'
echo ${GALYLEO_SCHEDULER}
if [[ ! -f "${job_name}.sh" ]]; then

slog append -f "${job_name}.sh" -m '#!/usr/bin/env sh'
Expand Down Expand Up @@ -624,6 +648,11 @@ function galyleo_launch() {
done
fi

# Run user init commands.
if [[ -n "${init_commands}" ]]; then
slog append -f "${job_name}.sh" -m "${init_commands}"
fi

# Activate a pre-installed conda environment specified by the user.
if [[ -n "${conda_env}" ]] && [[ -z "${conda_yml}" ]]; then
if [[ -n "${conda_init}" ]]; then
Expand Down Expand Up @@ -741,6 +770,9 @@ function galyleo_launch() {
# Submit Jupyter launch script to scheduler.
if [[ "${GALYLEO_SCHEDULER}" == 'slurm' ]]; then
job_id="$(sbatch ${job_name}.sh | grep -o '[[:digit:]]*')"
#cat ${job_name}.sh
#job_id=0 # don't run, just testing
#: # don't run, just testing
if [[ "${?}" -ne 0 ]]; then
slog error -m 'Failed job submission to Slurm.'
http_response="$(curl -s https://manage.${GALYLEO_REVERSE_PROXY_FQDN}/destroytoken.cgi?token=${REVERSE_PROXY_TOKEN})"
Expand All @@ -764,6 +796,7 @@ function galyleo_launch() {
# Associate batch job id to the connection token from the reverse proxy service.
http_response="$(curl -s https://manage.${GALYLEO_REVERSE_PROXY_FQDN}/linktoken.cgi?token=${REVERSE_PROXY_TOKEN}\&jobid=${job_id})"
slog output -m "${http_response}"
#: # don't run, just testing

# Always print to standard output the URL where the Jupyter notebook
# server may be accessed by the user.
Expand Down Expand Up @@ -974,6 +1007,7 @@ function galyleo_help() {
slog output -m " -B | --bind : ${singularity_bind_mounts}"
slog output -m " | --nv : ${singularity_gpu_type}"
slog output -m " -e | --env-modules : ${env_modules}"
slog output -m " | --init-commands : ${init_commands}"
slog output -m " | --conda-init : ${conda_init}"
slog output -m " | --conda-env : ${conda_env}"
slog output -m " | --conda-yml : ${conda_yml}"
Expand Down