Improve CLI testing (#4131)

.github/workflows/clang_linux/start.sh

@@ -4,22 +4,25 @@ set -e
 
 apt-get update -y
 DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    autoconf automake libtool make cmake ccache pkg-config python3-pip sqlite3 tar zip \
+    autoconf automake libtool make cmake ccache pkg-config python3-pip sqlite3 tar zip curl \
     clang++-10 jq python3-clang-10 \
     libsqlite3-dev \
     libtiff-dev libwebp-dev libzstd-dev \
     libcurl4-openssl-dev libnghttp2-dev libidn2-dev librtmp-dev libssh-dev \
-      libpsl-dev libssl-dev libkrb5-dev comerr-dev libldap2-dev libbrotli-dev \
+    libpsl-dev libssl-dev libkrb5-dev comerr-dev libldap2-dev libbrotli-dev \
     nlohmann-json3-dev libgtest-dev
 
-python3 -m pip install --user jsonschema
 export PATH=$HOME/.local/bin:$PATH
+python3 -m pip config --user set global.progress_bar off
+python3 -m pip install --user jsonschema pyyaml pytest
 
 cd "$WORK_DIR"
 
 if test -f "$WORK_DIR/ccache.tar.gz"; then
     echo "Restoring ccache..."
     (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+else
+    mkdir -p $HOME/.ccache
 fi
 
 ccache -M 500M

.github/workflows/linux_gcc_32bit/start.sh

@@ -12,15 +12,16 @@ dpkg --add-architecture i386
 apt-get update -y
 
 DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends  -o APT::Immediate-Configure=0  \
-    autoconf automake libtool make cmake ccache pkg-config python3-pip sqlite3 tar zip \
+    autoconf automake libtool make cmake ccache pkg-config python3-pip sqlite3 tar zip curl \
     gcc-multilib g++-multilib g++ jq dpkg-dev \
     libsqlite3-dev:$ARCH \
     libtiff-dev:$ARCH libwebp-dev:$ARCH libzstd-dev:$ARCH \
     libcurl4-openssl-dev:$ARCH libnghttp2-dev:$ARCH libidn2-dev:$ARCH librtmp-dev:$ARCH libssh-dev:$ARCH \
-      libpsl-dev:$ARCH libssl-dev:$ARCH libkrb5-dev:$ARCH comerr-dev:$ARCH libldap2-dev:$ARCH libbrotli-dev:$ARCH
+    libpsl-dev:$ARCH libssl-dev:$ARCH libkrb5-dev:$ARCH comerr-dev:$ARCH libldap2-dev:$ARCH libbrotli-dev:$ARCH
 
-python3 -m pip install --user jsonschema
 export PATH=$HOME/.local/bin:$PATH
+python3 -m pip config --user set global.progress_bar off
+python3 -m pip install --user jsonschema ruamel.yaml
 
 export CXXFLAGS='-m32 -D_GLIBCXX_ASSERTIONS'
 export CFLAGS='-m32'
@@ -36,6 +37,8 @@ cd "$WORK_DIR"
 if test -f "$WORK_DIR/ccache.tar.gz"; then
     echo "Restoring ccache..."
     (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+else
+    mkdir -p $HOME/.ccache
 fi
 
 ccache -M 500M

.github/workflows/mac/environment.yml

@@ -13,4 +13,6 @@ dependencies:
   - autoconf
   - automake
   - jsonschema
+  - ruamel.yaml
+  - pytest
 

.github/workflows/mingw_w64/start.sh

@@ -26,6 +26,8 @@ cd "$WORK_DIR"
 if test -f "$WORK_DIR/ccache.tar.gz"; then
     echo "Restoring ccache..."
     (cd $HOME && tar xzf "$WORK_DIR/ccache.tar.gz")
+else
+    mkdir -p $HOME/.ccache
 fi
 
 sudo apt-get install -y --no-install-recommends \
@@ -68,7 +70,7 @@ ln -s $MINGW_PREFIX/libgcc_s_seh-1.dll $WINE_SYSDIR
 ln -s /usr/$MINGW_ARCH/lib/libwinpthread-1.dll $WINE_SYSDIR
 
 # build zlib
-wget https://github.com/madler/zlib/archive/v1.2.11.tar.gz
+wget -q https://github.com/madler/zlib/archive/v1.2.11.tar.gz
 tar xzf v1.2.11.tar.gz
 (cd zlib-1.2.11 && \
   sudo make install -f win32/Makefile.gcc SHARED_MODE=1 \
@@ -79,13 +81,13 @@ tar xzf v1.2.11.tar.gz
 ln -s /usr/$MINGW_ARCH/bin/zlib1.dll $WINE_SYSDIR
 
 # build libtiff
-wget http://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz
+wget -q http://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz
 tar xzf tiff-4.1.0.tar.gz
 (cd tiff-4.1.0 && ./configure --host=$MINGW_ARCH --prefix=/usr/$MINGW_ARCH && make && sudo make install)
 ln -s /usr/$MINGW_ARCH/bin/libtiff-5.dll $WINE_SYSDIR
 
 # build sqlite3
-wget https://sqlite.org/2020/sqlite-autoconf-3330000.tar.gz
+wget -q https://sqlite.org/2020/sqlite-autoconf-3330000.tar.gz
 tar xzf sqlite-autoconf-3330000.tar.gz
 # Build with SQLITE_DQS=0 to ensure we properly use single quotes and double quotes (cf issue #2480)
 (cd sqlite-autoconf-3330000 &&

.github/workflows/windows.yml

@@ -37,6 +37,11 @@ jobs:
 
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - run: pip install pyyaml
+
       - uses: ilammy/msvc-dev-cmd@v1
 
       - name: Cache vcpkg packages
@@ -65,10 +70,17 @@ jobs:
             mkdir %PROJ_BUILD%
             cd %PROJ_BUILD%
             set PROJ_DIR=%GITHUB_WORKSPACE%\proj_dir
-            # Not directly linked to BUILD_SHARED_LIBS, but a way to test different C++ standard versions
+            :: Not directly linked to BUILD_SHARED_LIBS, but a way to test different C++ standard versions
             if "${{ env.BUILD_SHARED_LIBS }}"=="ON" (set EXTRA_CXX_FLAGS="/std:c++20")
             if "${{ env.BUILD_TYPE }}"=="Release" (set CMAKE_UNITY_BUILD_OPT="-DCMAKE_UNITY_BUILD=ON")
-            cmake -DCMAKE_BUILD_TYPE="${{ env.BUILD_TYPE }}" -DBUILD_SHARED_LIBS="${{ env.BUILD_SHARED_LIBS }}" -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX %EXTRA_CXX_FLAGS%" -DCMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX="%PROJ_DIR%" -DPROJ_DB_CACHE_DIR=%PROJ_DB_CACHE_DIR% %CMAKE_UNITY_BUILD_OPT% ..
+            cmake -D CMAKE_BUILD_TYPE="${{ env.BUILD_TYPE }}" ^
+              -D BUILD_SHARED_LIBS="${{ env.BUILD_SHARED_LIBS }}" ^
+              -D CMAKE_C_FLAGS="/WX" ^
+              -D CMAKE_CXX_FLAGS="/WX %EXTRA_CXX_FLAGS%" ^
+              -D CMAKE_TOOLCHAIN_FILE=c:/vcpkg/scripts/buildsystems/vcpkg.cmake ^
+              -D CMAKE_INSTALL_PREFIX="%PROJ_DIR%" ^
+              -D PROJ_DB_CACHE_DIR=%PROJ_DB_CACHE_DIR% ^
+              %CMAKE_UNITY_BUILD_OPT% ..
             ninja -v
             ninja install
             dir %PROJ_DIR%\bin
@@ -142,17 +154,28 @@ jobs:
             mingw-w64-x86_64-curl
             mingw-w64-x86_64-gcc
             mingw-w64-x86_64-libtiff
+            mingw-w64-x86_64-python-yaml
             mingw-w64-x86_64-sqlite3
 
       - name: Build
         run: |
+            mkdir -p $HOME/.ccache
             ccache -M 500M
             ccache -s
             PROJ_BUILD=${GITHUB_WORKSPACE}/build
             PROJ_DIR=${GITHUB_WORKSPACE}/proj_dir
             mkdir ${PROJ_BUILD}
             cd ${PROJ_BUILD}
-            cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DBUILD_SHARED_LIBS=${{ env.BUILD_SHARED_LIBS }} -DCMAKE_INSTALL_PREFIX="${PROJ_DIR}" -DCMAKE_C_FLAGS="-Werror" -DCMAKE_CXX_FLAGS="-Werror" -DUSE_CCACHE=ON -DCMAKE_UNITY_BUILD=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
+            cmake -D CMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
+              -D BUILD_SHARED_LIBS=${{ env.BUILD_SHARED_LIBS }} \
+              -D CMAKE_INSTALL_PREFIX="${PROJ_DIR}" \
+              -D CMAKE_C_FLAGS="-Werror" \
+              -D CMAKE_CXX_FLAGS="-Werror" \
+              -D USE_CCACHE=ON \
+              -D PROJ_DB_CACHE_DIR=$HOME/.ccache \
+              -D CMAKE_UNITY_BUILD=ON \
+              -D Python3_ROOT_DIR=/mingw64 \
+              ..
             make -j 2
             make install
             ls ${PROJ_DIR}/bin

cmake/ProjTest.cmake

@@ -1,5 +1,5 @@
 #
-# add test with sh script
+# Add command-line interface tests
 #
 
 function(proj_test_set_properties TESTNAME)
@@ -14,20 +14,28 @@ function(proj_test_set_properties TESTNAME)
     PROPERTY ENVIRONMENT ${_env})
 endfunction()
 
-function(proj_add_test_script_sh SH_NAME BIN_USE)
-  if(UNIX)
-    get_filename_component(testname ${SH_NAME} NAME_WE)
+function(proj_add_test_script_sh SH_NAME EXE_PATH)
+  set(testname ${SH_NAME})
 
-    add_test(NAME "${testname}"
-      WORKING_DIRECTORY ${PROJ_BINARY_DIR}/test/cli
-      COMMAND bash ${PROJ_SOURCE_DIR}/test/cli/${SH_NAME}
-      ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${BIN_USE}}
-    )
-    proj_test_set_properties(${testname})
-
-  endif()
+  add_test(NAME ${testname}
+    WORKING_DIRECTORY ${PROJ_BINARY_DIR}/test/cli
+    COMMAND bash ${PROJ_SOURCE_DIR}/test/cli/${SH_NAME} "${${EXE_PATH}}"
+  )
+  proj_test_set_properties(${testname})
 endfunction()
 
+function(proj_run_cli_test TESTFILE EXE_PATH)
+  set(testname ${TESTFILE})
+
+  add_test(NAME ${testname}
+    WORKING_DIRECTORY ${PROJ_BINARY_DIR}/test/cli
+    COMMAND ${Python3_EXECUTABLE}
+      ${PROJ_SOURCE_DIR}/test/cli/run_cli_test.py
+      --exe "${${EXE_PATH}}"
+      ${PROJ_SOURCE_DIR}/test/cli/${TESTFILE}
+  )
+  proj_test_set_properties(${testname})
+endfunction()
 
 function(proj_add_gie_test TESTNAME TESTCASE)
 

data/CMakeLists.txt

@@ -100,6 +100,10 @@ endforeach()
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/egm96_15_downsampled.gtx ${CMAKE_CURRENT_BINARY_DIR}/for_tests/egm96_15.gtx COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/ntv2_0_downsampled.gsb ${CMAKE_CURRENT_BINARY_DIR}/for_tests/ntv2_0.gsb COPYONLY)
 
+# test_cs2cs_datumfile has a special case
+execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/for_tests/dir with space")
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tests/conus "${CMAKE_CURRENT_BINARY_DIR}/for_tests/dir with space/myconus" COPYONLY)
+
 #
 #install
 #

scripts/fix_typos.sh

@@ -46,7 +46,7 @@ if ! test -d fix_typos; then
     )
 fi
 
-EXCLUDED_FILES="*configure,config.status,config.sub,*/autom4te.cache/*,libtool,aclocal.m4,depcomp,ltmain.sh,*.pdf,./m4/*,./fix_typos/*,./docs/build/*,./src/*generated*,./test/googletest/*,./include/proj/internal/nlohmann/json.hpp,*.before_reformat,./test/cli/test27,./test/cli/test83,./test/cli/pj_out27.dist,./test/cli/pj_out83.dist,geodesic.h,geodesic.c,geodtest.c,./docs/source/spelling_wordlist.txt"
+EXCLUDED_FILES="*configure,config.status,config.sub,*/autom4te.cache/*,libtool,aclocal.m4,depcomp,ltmain.sh,*.pdf,./m4/*,./fix_typos/*,./docs/build/*,./src/*generated*,./test/googletest/*,./include/proj/internal/nlohmann/json.hpp,*.before_reformat,geodesic.h,geodesic.c,geodtest.c,./docs/source/spelling_wordlist.txt"
 WORDS_WHITE_LIST="metres,als,lsat,twon,ang,PJD_ERR_LSAT_NOT_IN_RANGE,COLOR_GRAT,interm,Interm,Cartesian,cartesian,CARTESIAN,kilometre,centimetre,millimetre,millimetres,Australia,LINZ,LaTeX,BibTeX"
 
 python3 fix_typos/codespell/codespell.py -w -i 3 -q 2 -S $EXCLUDED_FILES \

src/4D_api.cpp

@@ -318,10 +318,10 @@ int pj_get_suggested_operation(PJ_CONTEXT *,
 
         if (spatialCriterionOK) {
             // The offshore test is for the "Test bug 245 (use +datum=carthage)"
-            // of testvarious. The long=10 lat=34 point belongs both to the
-            // onshore and offshore Tunisia area of uses, but is slightly
-            // onshore. So in a general way, prefer a onshore area to a
-            // offshore one.
+            // of test_cs2cs_various.yaml. The long=10 lat=34 point belongs
+            // both to the onshore and offshore Tunisia area of uses, but is
+            // slightly onshore. So in a general way, prefer a onshore area
+            // to a offshore one.
             if (iBest < 0 ||
                 (((alt.accuracy >= 0 && alt.accuracy < bestAccuracy) ||
                   // If two operations have the same accuracy, use

src/apps/cs2cs.cpp

@@ -388,7 +388,7 @@ int main(int argc, char **argv) {
     }
 
     /* This is just to check that pj_init() is locale-safe */
-    /* Used by nad/testvarious */
+    /* Used by test/cli/test_cs2cs_locale.sh */
     if (getenv("PROJ_USE_ENV_LOCALE") != nullptr)
         use_env_locale = 1;
 

src/iso19111/operation/coordinateoperationfactory.cpp

@@ -3276,7 +3276,7 @@ void CoordinateOperationFactory::Private::createOperationsWithDatumPivot(
     // ... but when transforming between 2 IGNF CRS, we do just one single pass
     // by allowing directly all transformation. There is no strong reason for
     // that particular case, except that otherwise we'd get different results
-    // for thest test/cli/testIGNF script when transforming a point outside
+    // for test/cli/test_cs2cs_ignf.yaml when transforming a point outside
     // the area of validity... Not totally sure the behaviour we try to preserve
     // here with the particular case is fundamentally better than the general
     // case. The general case is needed typically for the RGNC91-93 -> RGNC15