Python 3.12 update.

.github/workflows/bare.yaml

@@ -20,10 +20,14 @@ jobs:
     runs-on: windows-latest
     timeout-minutes: 45
 
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit requirements.txt
+      contents: write
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
     # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -36,6 +40,12 @@ jobs:
       timeout-minutes: 5
       run: bash ./build.sh test
 
+    # Commit changed requirements.txt back to the repository
+    - uses: chevah/git-auto-commit-action@HEAD
+      with:
+        commit_message: Automated update of requirements.txt from Windows build.
+        file_pattern: 'requirements.txt'
+
     # To use an RSA key with SFTPPlus, install upstream OpenSSH package,
     # which is more finicky in regards to file permissions.
     # Beware the commands in this step run under PowerShell.
@@ -84,7 +94,7 @@ jobs:
     runs-on: macos-13
     timeout-minutes: 90
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -94,7 +104,10 @@ jobs:
       run: |
         sudo mv -v /usr/local/bin/git{,.saved}
         sudo chmod -v a-r /usr/local/include/libintl.h
+        sudo chmod -v a-r /usr/local/include/gdbm.h
         sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.*
+        sudo chmod -v a-r /usr/local/opt/gdbm/lib/libgdbm*
+        sudo chmod -v a-r /usr/local/opt/mpdecimal/lib/libmpdec*
 
     - name: Build Pythia
       timeout-minutes: 30
@@ -103,8 +116,11 @@ jobs:
     # Fix back Homebrew, for working Shellcheck tests and tmate debugging.
     - name: Unhack Homebrew
       run: |
-        sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl*
         sudo chmod -v a+r /usr/local/include/libintl.h
+        sudo chmod -v a-r /usr/local/include/gdbm.h
+        sudo chmod -v a+r /usr/local/opt/gettext/lib/libintl*
+        sudo chmod -v a+r /usr/local/opt/gdbm/lib/libgdbm*
+        sudo chmod -v a+r /usr/local/opt/mpdecimal/lib/libmpdec*
         sudo mv -v /usr/local/bin/git{.saved,}
 
     - name: Test Pythia
@@ -133,7 +149,7 @@ jobs:
     runs-on: macos-latest
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
 

.github/workflows/docker.yaml

@@ -25,8 +25,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Alpine 3.12 has musl 1.1.24, Amazon 2 has glibc 2.26.
-        container: [ 'alpine:3.12', 'amazonlinux:2' ]
+        # Alpine Linux 3.15 has musl 1.2.2, Amazon Linux 2 has glibc 2.26.
+        container: [ 'alpine:3.15', 'amazonlinux:2' ]
     timeout-minutes: 60
     steps:
 
@@ -44,7 +44,13 @@ jobs:
       if: startsWith(matrix.container, 'amazonlinux')
       run: |
         yum -y upgrade
-        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu
+        yum -y install git-core gcc make m4 patch tar unzip perl perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu bzip2
+        # To avoid linking against libdb and gdmb libraries on Amazon Linux 2.
+        # Can't simply uninstall libdb-devel and gdmb-devel, they are required by perl-IPC-Cmd.
+        rm -v /usr/include/libdb/db.h
+        rm -v /usr/include/gdbm.h
+        # This is for avoiding building the NIS module with Python 3.12.
+        rm -v /usr/include/rpc/rpc.h
 
     - name: Ubuntu setup
       if: startsWith(matrix.container, 'ubuntu')

.gitignore

@@ -4,5 +4,5 @@
 /src/*/*.tar.gz
 /src/*/*.tgz
 /src/*/*.zip
-/build-pythia/
+/build-py3/
 /cache/

build.conf

@@ -9,22 +9,24 @@ DIST_DIR="dist"
 # Setting this as 0 disables tests during building (skips chevahbs_test phase).
 PYTHIA_BUILD_TESTS="${PYTHIA_BUILD_TESTS:-1}"
 
-# Python and lib versions.
-PYTHON_BUILD_VERSION="3.11.9"
+# Set versions for Python and its libraries.
+# For major Python updates, e.g. 3.11->3.12, also update pythia.conf and
+# src/Python-Windows/chevahbs (e.g. "python311._pth"->"python312._pth").
+PYTHON_BUILD_VERSION="3.12.7"
 LIBFFI_VERSION="3.4.6"
 ZLIB_VERSION="1.3.1"
 BZIP2_VERSION="1.0.8"
 # To check the signature of the XZ download:
 # "gpg --keyserver-options auto-key-retrieve --verify xz-*.sig xz-*.gz".
 XZ_VERSION="5.6.2"
 # Statically build the BSD libedit on selected platforms to get the
-# readline module available without linking to the GPL-only readline libs.
+# readline module available without linking to the GPL-only readline libraries.
 # If there's a need to reenable this, our libedit patch for Python 3.9 was
 # https://github.com/chevah/pythia/pull/5/commits/09c128154d23feb6b1a7cb5a8d79.
 # A newer patch is available at https://github.com/python/cpython/issues/57710.
 LIBEDIT_VERSION="20170329-3.1"
-# Our OpenSSL libs are only used for Python's "ssl" module lately.
-OPENSSL_VERSION="3.0.14"
+# Our OpenSSL libraries are only used for Python's "ssl" module lately.
+OPENSSL_VERSION="3.0.15"
 # Use the version of the "sqlite-autoconf-VERSION.tar.gz" upstream download.
 # To get its SHA3-256 signature: "openssl dgst -sha3-256 sqlite-autoconf-*".
 # When updating this, also update the year in src/sqlite/chevahbs, if needed.
@@ -36,19 +38,18 @@ SQLITE_VERSION="3460000"
 BOOTSTRAP_GET_PIP="https://bootstrap.pypa.io/get-pip.py"
 
 # Python modules installed after bootstraping pip.
-PIP_VERSION="24.0"
-SETUPTOOLS_VERSION="70.0.0"
+PIP_VERSION="24.2"
+SETUPTOOLS_VERSION="70.3.0"
 # pycparser is explicitly installed to work around setuptools auto dependencies.
 PYCPARSER_VERSION="2.22"
 
 # Python modules that have to be built and/or installed in Pythia.
-PSUTIL_VERSION="5.9.8"
 PYWIN32_VERSION="306"
+# To be removed when upstream builds a musl wheel for psutil.
+# More at https://github.com/giampaolo/psutil/pull/2126.
+PSUTIL_VERSION="6.0.0"
 
-# Pin safety to keep the deps of the test environment under control.
-SAFETY_VERSION="3.2.0"
-
-# Global flags for building required libs.
+# Global flags for building required libraries.
 BUILD_LIBFFI="no"
 BUILD_ZLIB="no"
 BUILD_BZIP2="yes"
@@ -72,7 +73,3 @@ PIP_ARGS=(\
     --index-url="$PIP_INDEX_URL" \
     --no-warn-script-location \
     )
-# Array of safety IDs to ignore.
-# 67599: pip 24.0, https://data.safetycli.com/v/67599/f17/ (disputed).
-# 70612: jinja2 3.1.4, https://data.safetycli.com/v/70612/97c (disputed).
-SAFETY_IGNORED_IDS=(67599 70612)

build.sh

@@ -8,6 +8,7 @@ set -o nounset    # always check if variables exist
 set -o errexit    # always exit on error
 set -o errtrace   # trap errors in functions as well
 set -o pipefail   # don't ignore exit codes when piping output
+set -o functrace  # inherit DEBUG and RETURN traps
 
 # Default PyPI server to use. Can be overwritten in build.conf.
 PIP_INDEX_URL="https://pypi.org/simple"
@@ -210,9 +211,12 @@ command_install_python_modules() {
         execute "$PYTHON_BIN" -m pip install "${PIP_ARGS[@]}" "$library"
     done
 
-    # When done, uninstall wheel.
+    echo "# Uninstalling wheel... #"
     execute "$PYTHON_BIN" -m pip uninstall --yes wheel
 
+    echo "# Regenerating requirements.txt file... #"
+    execute "$PYTHON_BIN" -m pip freeze --all > requirements.txt
+
     echo "::endgroup::"
 }
 
@@ -222,8 +226,6 @@ help_text_test="Run own tests for the newly-build Python distribution."
 command_test() {
     local test_file="test_python_binary_dist.py"
     local python_binary="$PYTHON_BIN"
-    local safety_id_to_ignore
-    declare -a safety_ignore_opts
 
     echo "::group::Chevah tests"
     if [ ! -d "$BUILD_DIR" ]; then
@@ -240,25 +242,12 @@ command_test() {
     execute cp src/chevah-python-tests/get_binaries_deps.sh "$BUILD_DIR"
     execute pushd "$BUILD_DIR"
     execute "$python_binary" "$test_file"
+    execute popd
     echo "::endgroup::"
 
     echo "::group::Security tests"
-    echo "## Testing for outdated packages and security issues... ##"
+    echo "## Testing for outdated packages... ##"
     execute "$python_binary" -m pip list --outdated --format=columns
-    execute "$python_binary" -m pip install "${PIP_ARGS[@]}" \
-        safety=="$SAFETY_VERSION"
-
-    if (( ${#SAFETY_IGNORED_IDS[@]} != 0 )); then
-        (>&2 echo "Following Safety DB IDs are excepted from checks:")
-        (>&2 echo -e "\t${SAFETY_IGNORED_IDS[*]}")
-        for safety_id_to_ignore in "${SAFETY_IGNORED_IDS[@]}"; do
-            safety_ignore_opts+=("-i $safety_id_to_ignore")
-        done
-    fi
-
-    execute "$python_binary" -m safety check --full-report \
-        "${safety_ignore_opts[@]}"
-    execute popd
     echo "::endgroup::"
 
     echo "::group::Shell tests"

functions_build.sh

@@ -170,7 +170,7 @@ build() {
 # Put stuff where it's expected and remove some of the cruft.
 #
 cleanup_install_dir() {
-    local python_lib_file="lib$PYTHON_VERSION.a"
+    local python_lib_file="lib${PYTHON_VERSION}.a"
 
     echo "::group::Clean up Python install dir"
     execute pushd "$BUILD_DIR/$PYTHON_BUILD_DIR"
@@ -243,6 +243,8 @@ cleanup_install_dir() {
                     execute mv pkgconfig/* lib/pkgconfig/
                     execute rmdir pkgconfig
                 fi
+                # Compress packaged Makefiles to save some space when unpacked.
+                execute bzip2 lib/config/Makefile*
                 ;;
         esac
         # Test that only bin/ and lib/ sub-dirs are left.

os_quirks.sh

@@ -101,9 +101,11 @@ case "$OS" in
 esac
 
 # Use PIC (Position Independent Code) with GCC on 64-bit arches (currently all).
-if [ "$CC" = "gcc" ]; then
-    export CFLAGS="${CFLAGS:-} -fPIC"
-fi
+case "$CC" in
+    gcc*)
+        export CFLAGS="${CFLAGS:-} -fPIC"
+        ;;
+esac
 
 # Get number of useful CPUs, to enable parallel builds where applicable.
 case "$OS" in

pkg_checks.sh

@@ -15,15 +15,15 @@
 # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh.
 
 # List of OS packages required for building Python/pyOpenSSL/cryptography etc.
-BASE_PKGS="gcc make m4 patch unzip perl"
+BASE_PKGS="gcc make m4 patch perl"
 if [ "$BUILD_LIBEDIT" = "yes" ]; then
     BASE_PKGS="$BASE_PKGS automake libtool"
 fi
-APK_PKGS="$BASE_PKGS git curl bash musl-dev linux-headers lddtree shadow \
-    openssh-client file unzip g++ musl-locales dejagnu"
-DEB_PKGS="$BASE_PKGS tar diffutils git curl \
+APK_PKGS="$BASE_PKGS git curl bash musl-dev linux-headers lddtree \
+    openssh-client file g++ musl-locales dejagnu"
+DEB_PKGS="$BASE_PKGS unzip tar diffutils git curl \
     openssh-client libtest-simple-perl xz-utils g++ dejagnu"
-RPM_PKGS="$BASE_PKGS tar diffutils git-core curl \
+RPM_PKGS="$BASE_PKGS bzip2 unzip tar diffutils git-core curl \
     openssh-clients perl-Test-Simple perl-IPC-Cmd xz gcc-c++ dejagnu"
 
 # Check for OS packages required for the build.
@@ -33,7 +33,7 @@ PACKAGES="$CC make m4 git patch curl sha512sum tar unzip"
 # This is defined as an array of commands and opts, to allow it to be quoted.
 CHECK_CMD=(command -v)
 
-# $CHECK_CMD should exit with 0 only when checked packages is installed.
+# $CHECK_CMD should exit with 0 only when checked package is installed.
 case "$OS" in
     windows)
         # Nothing to actually build on Windows.
@@ -54,7 +54,7 @@ case "$OS" in
         ;;
     linux*)
         if [ -x /sbin/apk ]; then
-            # Assumes Alpine Linux 3.12.
+            # Assumes Alpine Linux 3.15.
             CHECK_CMD=(apk info -q -e)
             PACKAGES="$APK_PKGS"
         elif [ -x /usr/bin/dpkg ]; then
@@ -74,7 +74,7 @@ esac
 # External checks with various exit codes are checked below.
 set +o errexit
 
-# If $CHECK_CMD is still (command -v), it's only a check for needed commands.
+# If $CHECK_CMD is still "(command -v)", it's only a check for needed commands.
 if [ -n "$PACKAGES" ]; then
     for package in $PACKAGES ; do
         echo "Checking if $package is available..."

pythia.conf

@@ -1,11 +1,14 @@
-PYTHON_CONFIGURATION="[email protected]"
+# When building a new major Python version, e.g. 3.11->3.12,
+# update this in advance (e.g. use "[email protected]"),
+# and remove BUILD_ENV_* files (e.g. with `./build.sh clean -a`).
+PYTHON_CONFIGURATION="[email protected]"
 # This is defined as a Bash array of options to be passed to commands.
-BASE_REQUIREMENTS=("chevah-brink==1.0.13" "paver==1.3.4")
+BASE_REQUIREMENTS=("chevah-brink==1.0.15" "paver==1.3.4")
+# Use our private PyPi server instead of the default one set in pythia.sh.
+PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple"
 # Use our production server instead of the GitHub releases set by default.
 BINARY_DIST_URI="https://bin.chevah.com:20443/production"
 # For testing packages, make sure this one is the last uncommented instance:
-#BINARY_DIST_URI="https://bin.chevah.com:20443/testing"
-# Also overwrite the default pypi.org site set by default in pythia.sh.
-PIP_INDEX_URL="https://bin.chevah.com:20443/pypi/simple"
-# This is used by the Python runtime.
+BINARY_DIST_URI="https://bin.chevah.com:20443/testing"
+# This directory is used by the Python runtime.
 CHEVAH_BUILD_DIR="build-py3"