Merge branch 'develop' into feature/pio_options

tests/atparse.bash

@@ -1,42 +1,106 @@
 #! /usr/bin/env bash
 function atparse {
-    local __set_x
-    [ -o xtrace ] && __set_x='set -x' || __set_x='set +x'
-    set +x
+    # Usage:
+    #     source atparse.bash # defines the "atparse" function; only do this once
+    #     atparse [ var1=value1 [ var2=value2 [...] ] ] < input_file > output_file
+    # This function filters text from stdin to stdout. It scans for text sequences like:
+    #    @[varname]
+    # And replaces them with the value of the corresponding ${varname} variable.
+    # You can provide variables that are not set in bash by providing them on the command line.
+    # If set -u is enabled, it will exit the process when a variable is empty or undefined via set -u.
+
     # Use __ in names to avoid clashing with variables in {var} blocks.
-    local __text __before __after __during
+    local __text # current line of text being parsed, or the current command-line argument being parsed
+    local __before # all text before the next @[...] option
+    local __after # all text after the next @[...] option
+    local __during # the contents of the @[...] option, including the @[ and ]
+    local __set_x=":" # will be "set -x" if the calling script had that option enabled
+    local __set_u=":" # will be "set -u" if the calling script had that option enabled
+    local __set_e=":" # will be "set -e" if the calling script had that option enabled
+    local __abort_on_undefined=NO # YES = script should abort if a variable is undefined, NO otherwise
+
+    # Ensure "set -x -e -u" are all inactive, but remember if they
+    # were active so we can reset them later.
+    if [[ -o xtrace ]] ; then
+        __set_x="set -x"
+    fi
+    if [[ -o errexit ]] ; then
+        __set_e="set -e"
+    fi
+    if [[ -o nounset ]] ; then
+        __set_u="set -u"
+        __abort_on_undefined=YES
+    fi
+    set +eux
+
+    # Allow setting variables on the atparse command line rather than the environment.
+    # They will be local variables in this function.
     for __text in "$@" ; do
         if [[ $__text =~ ^([a-zA-Z][a-zA-Z0-9_]*)=(.*)$ ]] ; then
             eval "local ${BASH_REMATCH[1]}"
             eval "${BASH_REMATCH[1]}="'"${BASH_REMATCH[2]}"'
         else
-            echo "ERROR: Ignoring invalid argument $__text\n" 1>&2
+            echo "ERROR: Ignoring invalid argument $__text" 1>&2
         fi
     done
-    while IFS= read -r __text ; do
+
+    # Loop over all lines of text.
+    while [[ 1 == 1 ]] ; do
+        # Read the next line of text. This will "fail" if no more text
+        # is left OR if the last line lacks an end-of-line character.
+        read -d '' -r __text
+
+        # Stop when "read" reports it is done ($? -ne 0) AND the text is
+        # non-empty (! -n "$__text"). This ensures we read the final line
+        # even if it lacks an end-of-line character.
+        if [[ $? -ne 0 ]] ; then
+            if [[ -n "$__text" ]] ; then
+                # Text remained, but it had no end-of-line.
+                :
+            else
+                break
+            fi
+        fi
+        # Search for strings like @[varname] or @['string'] or @[@]
         while [[ "$__text" =~ ^([^@]*)(@\[[a-zA-Z_][a-zA-Z_0-9]*\]|@\[\'[^\']*\'\]|@\[@\]|@)(.*) ]] ; do
             __before="${BASH_REMATCH[1]}"
             __during="${BASH_REMATCH[2]}"
             __after="${BASH_REMATCH[3]}"
-#            printf 'PARSE[%s|%s|%s]\n' "$__before" "$__during" "$__after"
             printf %s "$__before"
+            # @['string'] inserts string
             if [[ "$__during" =~ ^@\[\'(.*)\'\]$ ]] ; then
                 printf %s "${BASH_REMATCH[1]}"
+            # @[@] inserts @
             elif [[ "$__during" == '@[@]' ]] ; then
                 printf @
+            # @[varname] inserts $varname
             elif [[ "$__during" =~ ^@\[([a-zA-Z_][a-zA-Z_0-9]*)\] ]] ; then
+                # Flag unknown variables at this step only.
+                if [[ ${__abort_on_undefined} == YES ]] ; then
+                    set -u
+                fi
                 eval 'printf %s "$'"${BASH_REMATCH[1]}"'"'
+                if [[ ${__abort_on_undefined} == YES ]] ; then
+                    set +u
+                fi
+            # Unrecognized sequences are inserted verbatim.
             else
                 printf '%s' "$__during"
             fi
+            # Continue until we run out of text in this line.
             if [[ "$__after" == "$__text" ]] ; then
                 break
             fi
             __text="$__after"
         done
+        # Print the corrected text
         printf '%s\n' "$__text"
     done
+
+    # Restore the calling script's shell options.
     eval "$__set_x"
+    eval "$__set_u"
+    eval "$__set_e"
 }
 
 function test_atparse {
@@ -45,14 +109,18 @@ function test_atparse {
     testvar='[testvar]'
     var1='[var1]'
     var2='[var2]'
-    cat<<\EOF | atparse var3='**'
+    var4='[var4]'
+    ( cat<<\EOF ; echo -n "line with no end-of-line character [var4] = @[var4]" ) | atparse var3='**'
 Nothing special here. = @['Nothing special here.']
 [testvar] = @[testvar]
 [var1] [var2] = @[var1] @[var2]
 ** = @[var3]
+[var4] == @[var4]
 @ = @[@] = @['@']
+@[undefined_variable_that_should_exit_script_if_set_minus_u_is_used]
 -n
  eval "export PE$c=\${PE$c:-0}" = @[' eval "export PE$c=\${PE$c:-0}"']
 EOF
+    echo " ... this text should be on the same line as the line with no end-of-line character"
     echo "After block, \$var3 = \"$var3\" should be empty"
 }

tests/ci/repo_check.sh

@@ -1,111 +1,112 @@
 #!/bin/bash
 set -eu
 
-# This script checks if head repo of PR is up to date with ufs-weather-model develop
-# Checks for top level (ufs-weather-model) and next level components (submodules)
-result() {
-  if [[ -n $comment ]]; then
-    logID=$1
-    comment="@$logID please bring these up to date with respective authoritative repositories\n"$comment
-    printf %s "$comment"
-    #exit 1
-  fi
+get_shas () {
+    cwd=$(pwd)
+    # Get sha-1's of the top of develop and feature branches
+    app="Accept: application/vnd.github.v3+json"
+    url=$1
+    gitapi=$2
+    branch=$3
+    base_sha=$(curl -sS -H "$app" $gitapi | jq -r '.commit.sha')
+    workspace=$4
+    cd $workspace
+    git remote add upstream $url
+    git fetch -q upstream $branch
+    common=$(git merge-base $base_sha @)
+    echo $common $base_sha $workspace
+    if [[ $common != $base_sha ]]; then
+        printf "%s\n\n" "** $workspace **NOT** up to date"
+        flag_sync=false
+    fi
+    cd $cwd
 }
 
-# Declare variables
-declare -A base fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp-framework ccpp-physics atmos_cubed_sphere
-submodules="fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp-framework ccpp-physics atmos_cubed_sphere"
-comment=''
+flag_sync=true
+
 ownerID=$1
 
-# Base branch: this is the top of develop of ufs-weather-model
-base[repo]='https://github.com/ufs-community/ufs-weather-model'
-base[branch]='develop'
+declare -A urls branches pathes
+submodules="base fv3 mom6 cice ww3 stoch gocart cmeps cdeps hycom cmake ccpp_physics ccpp_framework aqm noahmp"
 
-# Submodules to check
-fv3[repo]='https://github.com/NOAA-EMC/fv3atm'
-fv3[branch]='develop'
-fv3[dir]='FV3'
+urls[base]='https://github.com/ufs-community/ufs-weather-model'
+branches[base]='develop'
+pathes[base]=''
 
-mom6[repo]='https://github.com/NOAA-EMC/MOM6'
-mom6[branch]='dev/emc'
-mom6[dir]='MOM6-interface/MOM6'
+urls[fv3]='https://github.com/NOAA-EMC/fv3atm'
+branches[fv3]='develop'
+pathes[fv3]='FV3'
 
-cice[repo]='https://github.com/NOAA-EMC/CICE'
-cice[branch]='emc/develop'
-cice[dir]='CICE-interface/CICE'
+urls[mom6]='https://github.com/NOAA-EMC/MOM6'
+branches[mom6]='dev/emc'
+pathes[mom6]='MOM6-interface/MOM6'
 
-ww3[repo]='https://github.com/NOAA-EMC/WW3'
-ww3[branch]='dev/ufs-weather-model'
-ww3[dir]='WW3'
+urls[cice]='https://github.com/NOAA-EMC/CICE'
+branches[cice]='emc/develop'
+pathes[cice]='CICE-interface/CICE'
 
-stoch[repo]='https://github.com/noaa-psl/stochastic_physics'
-stoch[branch]='master'
-stoch[dir]='stochastic_physics'
+urls[ww3]='https://github.com/NOAA-EMC/WW3'
+branches[ww3]='dev/ufs-weather-model'
+pathes[ww3]='WW3'
 
-gocart[repo]='https://github.com/GEOS-ESM/GOCART'
-gocart[branch]='develop'
-gocart[dir]='GOCART'
+urls[stoch]='https://github.com/noaa-psl/stochastic_physics'
+branches[stoch]='master'
+pathes[stoch]='stochastic_physics'
 
-cmeps[repo]='https://github.com/NOAA-EMC/CMEPS'
-cmeps[branch]='emc/develop'
-cmeps[dir]='CMEPS-interface/CMEPS'
+urls[gocart]='https://github.com/GEOS-ESM/GOCART'
+branches[gocart]='develop'
+pathes[gocart]='GOCART'
 
-cdeps[repo]='https://github.com/NOAA-EMC/CDEPS'
-cdeps[branch]='develop'
-cdeps[dir]='CDEPS-interface/CDEPS'
+urls[cmeps]='https://github.com/NOAA-EMC/CMEPS'
+branches[cmeps]='emc/develop'
+pathes[cmeps]='CMEPS-interface/CMEPS'
 
-hycom[repo]='https://github.com/NOAA-EMC/HYCOM-src'
-hycom[branch]='emc/develop'
-hycom[dir]='HYCOM-interface/HYCOM'
+urls[cdeps]='https://github.com/NOAA-EMC/CDEPS'
+branches[cdeps]='develop'
+pathes[cdeps]='CDEPS-interface/CDEPS'
 
-cmake[repo]='https://github.com/NOAA-EMC/CMakeModules'
-cmake[branch]='develop'
-cmake[dir]='CMakeModules'
+urls[hycom]='https://github.com/NOAA-EMC/HYCOM-src'
+branches[hycom]='emc/develop'
+pathes[hycom]='HYCOM-interface/HYCOM'
 
-ccpp-framework[repo]='https://github.com/NCAR/ccpp-framework'
-ccpp-framework[branch]='main'
-ccpp-framework[dir]='FV3/ccpp/framework'
+urls[cmake]='https://github.com/NOAA-EMC/CMakeModules'
+branches[cmake]='develop'
+pathes[cmake]='CMakeModules'
 
-ccpp-physics[repo]='https://github.com/ufs-community/ccpp-physics'
-ccpp-physics[branch]='ufs/dev'
-ccpp-physics[dir]='FV3/ccpp/physics'
+urls[ccpp_physics]='https://github.com/ufs-community/ccpp-physics'
+branches[ccpp_physics]='ufs/dev'
+pathes[ccpp_physics]='FV3/ccpp/physics'
 
-#upp[repo]='https://github.com/NOAA-EMC/UPP'
-#upp[branch]='develop'
-#upp[dir]='upp'
+urls[ccpp_framework]='https://github.com/NCAR/ccpp-framework'
+branches[ccpp_framework]='main'
+pathes[ccpp_framework]='FV3/ccpp/framework'
 
-atmos_cubed_sphere[repo]='https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere'
-atmos_cubed_sphere[branch]='dev/emc'
-atmos_cubed_sphere[dir]='FV3/atmos_cubed_sphere'
+urls[aqm]='https://github.com/NOAA-EMC/AQM'
+branches[aqm]='develop'
+pathes[aqm]='AQM'
 
-# Get sha-1's of the top of develop of ufs-weather-model
-app="Accept: application/vnd.github.v3+json"
-url="https://api.github.com/repos/ufs-community/ufs-weather-model/branches/develop"
-base[sha]=$(curl -sS -H "$app" $url | jq -r '.commit.sha')
-for submodule in $submodules; do
-  eval url=https://api.github.com/repos/ufs-community/ufs-weather-model/contents/'${'$submodule'[dir]}'
-  eval $submodule'[sha]=$(curl -sS -H "$app" $url | jq -r '.sha')'
-done
+urls[noahmp]='https://github.com/NOAA-EMC/noahmp'
+branches[noahmp]='develop'
+pathes[noahmp]='NOAHMP-interface/noahmp'
 
-# Check if the head branch is up to date with the base branch
-cd ${GITHUB_WORKSPACE}
-git remote add upstream ${base[repo]}
-git fetch -q upstream ${base[branch]}
-common=$(git merge-base ${base[sha]} @)
-if [[ $common != ${base[sha]} ]]; then
-  comment="* ufs-weather-model **NOT** up to date\n"
-fi
+#urls[upp]='https://github.com/NOAA-EMC/UPP'
+#branches[upp]='develop'
+#pathes[upp]='upp'
+
+#urls[cubed_sphere]='https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere'
+#branches[cubed_sphere]='dev/emc'
+#pathes[cubed_sphere]='FV3/atmos_cubed_sphere'
 
 for submodule in $submodules; do
-  eval cd ${GITHUB_WORKSPACE}/'${'$submodule'[dir]}'
-  eval git remote add upstream '${'$submodule'[repo]}'
-  eval git fetch -q upstream '${'$submodule'[branch]}'
-  common=$(eval git merge-base '${'$submodule'[sha]}' @)
-  if (eval test $common != '${'$submodule'[sha]}'); then
-    comment+="* $submodule **NOT** up to date\n"
-  fi
+    url=${urls[$submodule]}
+    branch=${branches[$submodule]}
+    workspace=${GITHUB_WORKSPACE}'/'${pathes[$submodule]}
+    gitapi=$(echo "$url" | sed 's/github.com/api.github.com\/repos/g')'/branches/'$branch
+    get_shas $url $gitapi $branch $workspace
 done
 
-result $ownerID
-exit 0
+if [[ $flag_sync=='true' ]]; then
+    exit 0
+else
+    exit 0
+fi