diff --git a/ci/cases/C48_S2S.yaml b/ci/cases/C48_S2S.yaml index c1c59b5659..00d391559d 100644 --- a/ci/cases/C48_S2S.yaml +++ b/ci/cases/C48_S2S.yaml @@ -3,10 +3,11 @@ experiment: mode: forecast-only arguments: + pslot: ${pslot} app: S2S resdet: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR idate: 2021032312 edate: 2021032312 yaml: ${HOMEgfs_PR}/ci/platforms/gfs_defaults_ci-updates.yaml diff --git a/ci/cases/C96C48_hybatmDA.yaml b/ci/cases/C96C48_hybatmDA.yaml index 6a0e41fbe1..660291a3be 100644 --- a/ci/cases/C96C48_hybatmDA.yaml +++ b/ci/cases/C96C48_hybatmDA.yaml @@ -3,11 +3,12 @@ experiment: mode: cycled arguments: + pslot: ${pslot} app: ATM resdet: 96 resens: 48 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR icsdir: ${ICSDIR_ROOT}/C96C48 idate: 2021122018 edate: 2021122106 diff --git a/ci/cases/C96_atm3DVar.yaml b/ci/cases/C96_atm3DVar.yaml index 5a93bd6454..f103f08d1f 100644 --- a/ci/cases/C96_atm3DVar.yaml +++ b/ci/cases/C96_atm3DVar.yaml @@ -3,10 +3,11 @@ experiment: mode: cycled arguments: + pslot: ${pslot} app: ATM resdet: 96 - comrot: ${RUNTESTS}/${pslot}/COMROT - expdir: ${RUNTESTS}/${pslot}/EXPDIR + comrot: ${RUNTESTS}/COMROT + expdir: ${RUNTESTS}/EXPDIR icsdir: ${ICSDIR_ROOT}/C96C48 idate: 2021122018 edate: 2021122106 diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 40cd3a7c3e..fd2e579410 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -6,7 +6,6 @@ set -eux # running rocotostat on each to determine if the experiment has # succeeded or faild. This script is intended # to run from within a cron job in the CI Managers account -# Abstract TODO ##################################################################################### HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" @@ -78,11 +77,11 @@ for pr in ${pr_list}; do if [[ ! -d "${pr_dir}/RUNTESTS" ]]; then continue fi - num_cases=$(find "${pr_dir}/RUNTESTS" -mindepth 1 -maxdepth 1 -type d | wc -l) || true - #Check for PR success when ${pr_dir}/RUNTESTS is void of subfolders + #Check for PR success when ${pr_dir}/RUNTESTS/EXPDIR is void of subfolders # since all successfull ones where previously removed - if [[ "${num_cases}" -eq 0 ]] && [[ -d "${pr_dir}/RUNTESTS" ]]; then + # shellcheck disable=SC2312 + if [[ -z $(ls -A "${pr_dir}/RUNTESTS/EXPDIR") ]] ; then "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Running" --add-label "CI-${MACHINE_ID^}-Passed" sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}" @@ -92,14 +91,14 @@ for pr in ${pr_list}; do continue fi - for cases in "${pr_dir}/RUNTESTS/"*; do - pslot=$(basename "${cases}") || true + for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do + pslot=$(basename "${pslot_dir}") || true if [[ -z "${pslot+x}" ]]; then - echo "No cases found in ${pr_dir}/RUNTESTS .. exiting" + echo "No experiments found in ${pslot_dir} .. exiting" exit 0 fi - xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml" - db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db" + xml="${pslot_dir}/${pslot}.xml" + db="${pslot_dir}/${pslot}.db" if [[ ! -f "${db}" ]]; then continue fi @@ -138,7 +137,8 @@ for pr in ${pr_list}; do sed -i "s/\`\`\`//2g" "${GFS_CI_ROOT}/PR/${pr}/output_${id}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${GFS_CI_ROOT}/PR/${pr}/output_${id}" #Remove Experment cases that completed successfully - rm -Rf "${pr_dir}/RUNTESTS/${pslot}" + rm -Rf "${pslot_dir}" + rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}" fi done done diff --git a/ci/scripts/clone-build_ci.sh b/ci/scripts/clone-build_ci.sh index 2adab196e0..83f43ef54b 100755 --- a/ci/scripts/clone-build_ci.sh +++ b/ci/scripts/clone-build_ci.sh @@ -78,7 +78,8 @@ echo "${commit}" > "../commit" # run checkout script cd sorc || exit 1 set +e -./checkout.sh -c -g -u >> log.checkout 2>&1 +# TODO enable -u later when GDASApp tests are added +./checkout.sh -c -g >> log.checkout 2>&1 checkout_status=$? if [[ ${checkout_status} != 0 ]]; then { diff --git a/ci/scripts/create_experiment.py b/ci/scripts/create_experiment.py index c8b95aaf0e..04a4b83cea 100755 --- a/ci/scripts/create_experiment.py +++ b/ci/scripts/create_experiment.py @@ -20,6 +20,7 @@ with an error code of 0 upon success. """ +import os from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path @@ -80,7 +81,6 @@ def input_args(): setup_expt_args = YAMLFile(path=user_inputs.yaml) HOMEgfs = Path.absolute(Path(user_inputs.dir)) - pslot = Path(user_inputs.yaml).stem type = setup_expt_args.experiment.type mode = setup_expt_args.experiment.mode @@ -93,9 +93,6 @@ def input_args(): setup_expt_cmd.add_default_arg(f'--{conf}') setup_expt_cmd.add_default_arg(str(value)) - setup_expt_cmd.add_default_arg('--pslot') - setup_expt_cmd.add_default_arg(pslot) - logger.info(f'Run command: {setup_expt_cmd.command}') setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr')) setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout')) @@ -103,7 +100,7 @@ def input_args(): setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr) setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py')) - expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(pslot))) + expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(setup_expt_args.arguments.pslot))) setup_xml_cmd.add_default_arg(str(expdir)) logger.info(f'Run command: {setup_xml_cmd.command}') diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 1bf77537fa..0e1f61730b 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -148,11 +148,15 @@ for pr in ${pr_list}; do ############################################################# HOMEgfs_PR="${pr_dir}/global-workflow" export HOMEgfs_PR + cd "${HOMEgfs_PR}" + pr_sha=$(git rev-parse --short HEAD) + for yaml_config in "${HOMEgfs_PR}/ci/cases/"*.yaml; do - pslot=$(basename "${yaml_config}" .yaml) || true + case=$(basename "${yaml_config}" .yaml) || true + pslot="${case}_${pr_sha}" export pslot set +e - "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${pslot}.yaml" --dir "${HOMEgfs_PR}" + "${HOMEgfs}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/${case}.yaml" --dir "${HOMEgfs_PR}" ci_status=$? set -e if [[ ${ci_status} -eq 0 ]]; then diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index 116c7f3a7f..5740b2955a 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -69,8 +69,8 @@ for pr in ${pr_list}; do continue fi num_cases=0 - for cases in "${pr_dir}/RUNTESTS/"*; do - if [[ ! -d "${cases}" ]]; then + for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do + if [[ ! -d "${pslot_dir}" ]]; then continue fi ((num_cases=num_cases+1)) @@ -78,10 +78,10 @@ for pr in ${pr_list}; do if [[ "${num_cases}" -gt "${max_concurrent_cases}" ]]; then continue fi - pslot=$(basename "${cases}") - xml="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.xml" - db="${pr_dir}/RUNTESTS/${pslot}/EXPDIR/${pslot}/${pslot}.db" - echo "Running: ${rocotorun} -v 6 -w ${xml} -d ${db}" + pslot=$(basename "${pslot_dir}") + xml="${pslot_dir}/${pslot}.xml" + db="${pslot_dir}/${pslot}.db" + echo "Running: ${rocotorun} -v 10 -w ${xml} -d ${db}" "${rocotorun}" -v 10 -w "${xml}" -d "${db}" done done