Remove RUNDIRS before running CI cases to cover re-run events (NOAA-E…

…MC#3076)

This Pr removes **pslot** dir (with hash) from local archive folder on
CI case completion and also adds `cleanup_experiment` function in the
BASH utilities to clean up after a CI case runs. This change
consolidates cleanup actions and adds functionality to clean the local
archive folders.

Key changes:

- Fixes bug for cleaning up the `RUNDIRS` directory before new
experiments are run in the case the pipeline is being ran for a second
time
- Updated `$HOMEgfs/ci/scripts/check_ci.sh` to use the new
`cleanup_experiment` function under the BASH CI system.
- Added `cleanup_experiment `function in
`$HOMEgfs/ci/scripts/utils/ci_utils.sh` to handle various cleanups
including:
  - COMROOT/EXPDIR per case
  - ARCDIR and ATADIR directories for archive
  - STMP/RUNDIRS/${PSLOT}
- Added get config var from EXPDIR utility
[get_config_var.py](https://github.com/NOAA-EMC/global-workflow/pull/2961/files#diff-b780733ae1d45917730364e09a6c510e79d4cc8cffad6d9020c7961c53b987bc)
for getting config values in BASH

Resolves NOAA-EMC#2954
Resolves NOAA-EMC#3066

ci/Jenkinsfile

@@ -222,6 +222,7 @@ pipeline {
                                                 def build_system = yaml_case.experiment.system
                                                 try {
                                                     sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {

ci/scripts/check_ci.sh

@@ -168,8 +168,7 @@ for pr in ${pr_list}; do
     fi
     if [[ "${rocoto_state}" == "DONE" ]]; then
       #Remove Experment cases that completed successfully
-      rm -Rf "${pslot_dir}"
-      rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
+      "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh" cleanup_experiment "${pslot_dir}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
       DATE=$(date +'%D %r')

ci/scripts/utils/ci_utils.sh

@@ -122,7 +122,9 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  # Remove RUNDIRS dir incase this is a retry
+  # Remove RUNDIRS dir incase this is a retry (STMP now in host file)
+  STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s)
+  echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry"
   rm -Rf "${STMP}/RUNDIRS/${pslot}"
 
   "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
@@ -137,7 +139,6 @@ function publish_logs() {
     local PR_header="$1"
     local dir_path="$2"
     local file="$3"
-
     local full_paths=""
     while IFS= read -r line; do
         full_path="${dir_path}/${line}"
@@ -155,3 +156,24 @@ function publish_logs() {
     fi
     echo "${URL}"
 }
+
+function cleanup_experiment() {
+
+    local EXPDIR="$1"
+    local pslot
+    local ARCDIR
+    local ATARDIR
+    local COMROOT
+
+    EXPDIR="$1"
+    pslot=$(basename "${EXPDIR}")
+
+    # Use the Python utility to get the required variables
+    read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true
+
+    rm -Rf "${ARCDIR:?}"
+    rm -Rf "${ATARDIR:?}"
+    rm -Rf "${COMROOT}/${pslot:?}"
+    rm -Rf "${EXPDIR}/${pslot:?}"
+    rm -Rf "${STMP}/RUNDIRS/${pslot:?}"
+}

ci/scripts/utils/get_config_var.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import os
+import argparse
+from wxflow import Configuration
+
+
+def get_config_vars(var_names, config_path):
+    """
+    GET_CONFIG_VARS Get configuration variables from a config file or directory.
+    Parameters:
+    var_names (list of str): The names of the configuration variables to retrieve.
+    config_path (str): The path to the configuration file or directory.
+    Returns:
+    list of str: The values of the specified configuration variables.
+    """
+    if os.path.isfile(config_path):
+        config_dir = os.path.dirname(config_path)
+        config_file = os.path.basename(config_path)
+    elif os.path.isdir(config_path):
+        config_dir = config_path
+        config_file = 'config.base'
+    config = Configuration(config_dir)
+    config_data = config.parse_config(config_file)
+    return [config_data[var_name] for var_name in var_names]
+
+
+if __name__ == "__main__":
+    """
+    Main entry point for the script.
+    Parses command-line arguments and retrieves the specified configuration variables.
+    """
+    parser = argparse.ArgumentParser(description="Get configuration variables from a config file or directory.")
+    parser.add_argument("var_names", nargs='+', help="The names of the configuration variables to retrieve.")
+    parser.add_argument("config_path", help="The path to the configuration file or directory.")
+
+    args = parser.parse_args()
+
+    var_names = args.var_names
+    config_path = args.config_path
+
+    values = get_config_vars(var_names, config_path)
+    print(" ".join(values))