Feature #2379 develop sonarqube updates (#2850)

* Per #2379, move rgb2ctable.py into the python utility scripts directory for better organization and to enable convenient SonarQube scanning.

* Per #2379, remove point.py from the vx_python3_utils directory which cleary was inadvertenlty added during development 4 years ago. As far as I can tell it isn't being called by any other code and doesn't belong in the repository. Note that scripts/python/met/point.py has the same name but is entirely different.

* Per #2379, update the GHA SonarQube scan to do a single one with Python and C++ combined. The nightly build script is still doing 2 separate scans for now. If this all works well, they could also be combined into a single one.

* Per #2379, eliminate MET_CONFIG_OPTIONS from the SonarQube workflow since it doesn't need to be and probably shouldn't be configurable.

* Per #2379, trying to copy report-task.txt out of the image

* Per #2379, update build_met_sonarqube.sh to check the scan return status

* Per #2379, fix bash assignment syntax

* Per #2379, remove unused SCRIPT_DIR envvar

* Per #2379, switch to a single SonarQube scan for MET's nightly build as well

.github/jobs/build_sonarqube_image.sh

@@ -28,7 +28,6 @@ time_command docker build -t ${DOCKERHUB_TAG} \
     --build-arg MET_BASE_REPO \
     --build-arg MET_BASE_TAG \
     --build-arg SOURCE_BRANCH \
-    --build-arg MET_CONFIG_OPTS \
     --build-arg SONAR_SCANNER_VERSION \
     --build-arg SONAR_HOST_URL \
     --build-arg SONAR_TOKEN \
@@ -41,6 +40,6 @@ fi
 
 # Copy the .scannerwork directory from the image
 id=$(docker create ${DOCKERHUB_TAG})
-time_command docker cp $id:/met/.scannerwork /tmp/met_scannerwork 
+time_command mkdir -p /tmp/scannerwork
+time_command docker cp $id:/met/.scannerwork/report-task.txt /tmp/scannerwork/report-task.txt
 docker rm -v $id
-

.github/workflows/sonarqube.yml

@@ -64,26 +64,15 @@ jobs:
           MET_BASE_TAG: v3.2
           SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}
           WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }}
-          MET_CONFIG_OPTS: '--enable-all'
           SONAR_SCANNER_VERSION: 5.0.1.3006
           SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
-      - name: SonarQube Python Quality Gate check
-        id: sonarqube-python-quality-gate-check
+      - name: SonarQube Quality Gate check
+        id: sonarqube-quality-gate-check
         uses: sonarsource/sonarqube-quality-gate-action@master
         with:
-          scanMetadataReportFile: /tmp/met_scannerwork/python-report-task.txt
-        timeout-minutes: 5
-        env:
-          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-
-      - name: SonarQube CXX Quality Gate check
-        id: sonarqube-cxx-quality-gate-check
-        uses: sonarsource/sonarqube-quality-gate-action@master
-        with:
-          scanMetadataReportFile: /tmp/met_scannerwork/cxx-report-task.txt
+          scanMetadataReportFile: /tmp/scannerwork/report-task.txt
         timeout-minutes: 5
         env:
           SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
@@ -100,4 +89,3 @@ jobs:
           name: logs_sonarqube
           path: ${{ runner.workspace }}/logs
           if-no-files-found: ignore
-

internal/scripts/docker/Dockerfile.sonarqube

@@ -92,4 +92,3 @@ RUN if [ ! -e "${MET_REPO_DIR}/configure.ac" ]; then \
 
 RUN cd ${MET_REPO_DIR} \
  && internal/scripts/docker/build_met_sonarqube.sh
-

internal/scripts/docker/build_met_sonarqube.sh

@@ -5,7 +5,7 @@
 #
 # This build_met_sonarqube.sh script must be run from the top-level
 # directory of the MET repository to be analyzed. It runs SonarQube to
-# scan both the Python and C/C++ MET source code.
+# scan the MET source code.
 #
 # Usage: internal/scripts/docker/build_met_sonarqube.sh
 #
@@ -48,12 +48,6 @@ if [ -z ${SONAR_REFERENCE_BRANCH+x} ]; then
   exit 1
 fi
 
-# Check whether MET_CONFIG_OPTS is defined
-if [ -z ${MET_CONFIG_OPTS+x} ]; then
-  MET_CONFIG_OPTS='--enable-all'
-  echo "Setting MET_CONFIG_OPTS=${MET_CONFIG_OPTS} to scan all available options."
-fi
-
 # Locate the wrapper
 WRAPPER_NAME=build-wrapper-linux-x86-64
 SONAR_WRAPPER=$(which $WRAPPER_NAME 2> /dev/null)
@@ -81,52 +75,33 @@ if [ -z ${SONARQUBE_OUT_DIR} ]; then
   export SONARQUBE_OUT_DIR=bw-outputs
 fi
 
+# Define the version string
+SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ')
+
 # Store the full path to the scripts directory
 SONAR_PROPERTIES_DIR=internal/scripts/sonarqube
 SONAR_PROPERTIES=sonar-project.properties
 
-# Copy sonar-project.properties for Python code
+# Configure the sonar-project.properties
 [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
-sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
+sed -e "s|SONAR_PROJECT_KEY|MET-GHA|" \
+    -e "s|SONAR_PROJECT_NAME|MET GHA|" \
+    -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \
     -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
-    -e "s|SONAR_PROJECT_KEY|MET-GHA-Python|" \
-    -e "s|SONAR_PROJECT_NAME|MET GHA Python|" \
+    -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
     -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \
-    -e "s|SONAR_REFERENCE_BRANCH|$SONAR_REFERENCE_BRANCH|" \
-    $SONAR_PROPERTIES_DIR/python.sonar-project.properties > $SONAR_PROPERTIES
+    $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES
 
 # The source and reference branches must differ to define new code
 if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then
   echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES
 fi
-
-# Run SonarQube scan for Python code
-time_command $SONAR_SCANNER
-
-# Copy the Python scan report-task.txt file
-mkdir -p /met/.scannerwork
-cp .scannerwork/report-task.txt /met/.scannerwork/python-report-task.txt
-
-# Copy sonar-project.properties for C/C++ code
-[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
-sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
-    -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
-    -e "s|SONAR_PROJECT_KEY|MET-GHA-CXX|" \
-    -e "s|SONAR_PROJECT_NAME|MET GHA CXX|" \
-    -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \
-    -e "s|SONAR_REFERENCE_BRANCH|$SONAR_REFERENCE_BRANCH|" \
-    $SONAR_PROPERTIES_DIR/sonar-project.properties > $SONAR_PROPERTIES
 
-# The source and reference branches must differ to define new code
-if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then
-  echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES
-fi
-
-# Run the configure script
+# Run the MET configure script
 time_command ./configure \
   BUFRLIB_NAME=${BUFRLIB_NAME} \
   GRIB2CLIB_NAME=${GRIB2CLIB_NAME} \
-  ${MET_CONFIG_OPTS} \
+  --enable-all \
   CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" \
   LIBS="-ltirpc"
 
@@ -136,11 +111,18 @@ time_command make clean
 # Run SonarQube make
 time_command $SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make
 
-# Run SonarQube scan for C/C++ code
+# Run SonarQube scan
 time_command $SONAR_SCANNER
+status=$?
+
+# Check return status
+if [[ $status -ne 0 ]]; then
+  echo "ERROR: ${0} -> the SonarQube scan returned with non-zero status (${status})!"
+  exit ${status}
+fi
 
-# Copy the C/C++ scan report-task.txt file
+# Copy the scan report-task.txt file
 mkdir -p /met/.scannerwork
-cp .scannerwork/report-task.txt /met/.scannerwork/cxx-report-task.txt
+cp .scannerwork/report-task.txt /met/.scannerwork/report-task.txt
 
 [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES

internal/scripts/sonarqube/run_sonarqube.sh

@@ -24,10 +24,10 @@
 GIT_REPO="https://github.com/dtcenter/${GIT_REPO_NAME}"
 
 function usage {
-        echo
-        echo "USAGE: $(basename $0) name"
-        echo "   where \"name\" specifies a branch, tag, or hash."
-        echo
+  echo
+  echo "USAGE: $(basename $0) name"
+  echo "   where \"name\" specifies a branch, tag, or hash."
+  echo
 }
 
 # Check for arguments
@@ -90,7 +90,6 @@ function run_command() {
   return ${STATUS}
 }
 
-
 # Store the full path to the scripts directory
 SCRIPT_DIR=`dirname $0`
 if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi 
@@ -113,48 +112,35 @@ export MET_DEVELOPMENT=true
 # Run the configure script
 run_command "./configure --prefix=`pwd` --enable-all"
 
-# Set the build id
-#BUILD_ID="MET-${1}"
+# Define the version string
+SONAR_PROJECT_VERSION=$(grep "^version" docs/conf.py | cut -d'=' -f2 | tr -d "\'\" ")
 
 SONAR_PROPERTIES=sonar-project.properties
 
-# Copy sonar-project.properties for Python code
+# Configure sonar-project.properties
 [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
 [ -z "$SONAR_HOST_URL" ] && SONAR_HOST_URL="http://localhost:9000"
 if [ -z "$SONAR_TOKEN" ]; then
   echo "  == ERROR == SONAR_TOKEN is not defined"
   exit 1
 else
-  sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
-      -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
-      -e "s|SONAR_PROJECT_KEY|MET_python_NB|" \
-      -e "s|SONAR_PROJECT_NAME|MET python Nightly Build|" \
-      -e "s|SONAR_BRANCH_NAME|develop|" \
-      $SCRIPT_DIR/python.sonar-project.properties > $SONAR_PROPERTIES
-
-  # Run SonarQube scan for Python code
-  run_command "$SONAR_SCANNER"
-
-  # Copy sonar-project.properties for C/C++ code
   [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
-  sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
-      -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
-      -e "s|SONAR_PROJECT_KEY|MET_develop_NB|" \
+  sed -e "s|SONAR_PROJECT_KEY|MET_NB|" \
       -e "s|SONAR_PROJECT_NAME|MET Nightly Build|" \
-      -e "s|SONAR_BRANCH_NAME|develop|" \
-      $SCRIPT_DIR/sonar-project.properties > $SONAR_PROPERTIES
+      -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \
+      -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
+      -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
+      -e "s|SONAR_BRANCH_NAME|${1}|" \
+      $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES
 
   # Run SonarQube clean
   run_command "make clean"
 
-  # Run SonarQube make
+  # Run SonarQube build wrapper
   run_command "$SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make"
 
-  # Run SonarQube scan for C/C++ code
+  # Run SonarQube scan
   run_command "$SONAR_SCANNER"
 
   [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
 fi
-
-# Run SonarQube report generator to make a PDF file
-#TODAY=`date +%Y%m%d`

internal/scripts/sonarqube/sonar-project.properties

@@ -1,17 +1,15 @@
+# Project and source code settings
 sonar.projectKey=SONAR_PROJECT_KEY
 sonar.projectName=SONAR_PROJECT_NAME
-sonar.projectVersion=1.0
-
-sonar.sources=src
+sonar.projectVersion=SONAR_PROJECT_VERSION
+sonar.branch.name=SONAR_BRANCH_NAME
+sonar.sources=src,scripts/python,data/wrappers
+sonar.python.version=3.6.3
+sonar.sourceEncoding=UTF-8
 
 # The build-wrapper output dir
 sonar.cfamily.build-wrapper-output=bw-outputs
 
-# Encoding of the source files
-sonar.sourceEncoding=UTF-8
-
-#----- Default SonarQube server
+# SonarQube server
 sonar.host.url=SONAR_HOST_URL
-
 sonar.token=SONAR_TOKEN
-sonar.branch.name=SONAR_BRANCH_NAME