conformance.yml

# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Workflow for running conformance tests.

name: Conformance
on:
  merge_group:
  push:
    branches:
      - main
      - v*.*.x
  pull_request:
    types: [opened, reopened, labeled, unlabeled, synchronize]

permissions:
  contents: read

env:
  CONFORMANCE_REPO_HASH: a3ee00672dd1b58c2b5a7d5e8b9e4a7b9e53ec1e
  LIBJXL_VERSION: "0.10.2"
  LIBJXL_ABI_VERSION: "0.10"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  warmup: # If necessary, fetch files just once, before tests are run.
    name: Warmup caches
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI:none') }}
    runs-on: ubuntu-latest
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
      with:
        egress-policy: audit

    - name: Checkout the conformance source
      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      with:
        repository: libjxl/conformance
        ref: ${{ env.CONFORMANCE_REPO_HASH }}
        path: conformance
    - name: Cache
      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
      with:
        path: ${{ github.workspace }}/conformance/.objects
        key: conformance-refs
    - name: Download and link conformance files
      run: |
        ${{ github.workspace }}/conformance/scripts/download_and_symlink.sh

  build:
    name: Conformance Build ${{ matrix.name }}
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI:none') }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: AVX3
            cflags: -DHWY_DISABLED_TARGETS=HWY_AVX3-1
          - name: AVX2
            cflags: -DHWY_DISABLED_TARGETS=HWY_AVX2-1
          - name: SSE4
            cflags: -DHWY_DISABLED_TARGETS=HWY_SSE4-1
          - name: SSSE3
            cflags: -DHWY_DISABLED_TARGETS=HWY_SSSE3-1
          - name: EMU128
            cflags: -DHWY_COMPILE_ONLY_EMU128=1
          - name: SCALAR
            cflags: -DHWY_COMPILE_ONLY_SCALAR=1
          - name: SCALAR_ASAN
            cflags: -DHWY_COMPILE_ONLY_SCALAR=1
            build_type: asan
    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
      with:
        egress-policy: audit

    - name: Install build deps
      run: |
        sudo rm -f /var/lib/man-db/auto-update
        sudo apt update
        sudo apt install -y \
          ccache \
          clang \
          cmake \
          doxygen \
          graphviz \
          libbenchmark-dev \
          libbenchmark-tools \
          libbrotli-dev \
          libgdk-pixbuf2.0-dev \
          libgif-dev \
          libgtest-dev \
          libgtk2.0-dev  \
          libjpeg-dev \
          libopenexr-dev \
          libpng-dev \
          libwebp-dev \
          ninja-build \
          pkg-config \
          xvfb \
          ${{ matrix.apt_pkgs }} \
        #
        echo "CC=clang" >> $GITHUB_ENV
        echo "CXX=clang++" >> $GITHUB_ENV
    - name: Checkout the jxl source
      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      with:
        submodules: true
        fetch-depth: 2
    - name: Git environment
      id: git-env
      run: |
        echo "parent=$(git rev-parse ${{ github.sha }}^)" >> $GITHUB_OUTPUT
      shell: bash
    - name: ccache
      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
      with:
        path: ${{ env.CCACHE_DIR }}
        # When the cache hits the key it is not updated, so if this is a rebuild
        # of the same Pull Request it will reuse the cache if still around. For
        # either Pull Requests or new pushes to main, this will use the parent
        # hash as the starting point from the restore-keys entry.
        key: conformance-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }}
        restore-keys: |
          conformance-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }}
    - name: Build
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        CMAKE_FLAGS="${{ matrix.cflags }}" \
        TARGETS="tools/djxl" \
        ./ci.sh ${{ matrix.build_type || 'release' }} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
          -DBUILD_TESTING=OFF
        # Flatten the artifacts directory structure
        cp tools/conformance/conformance.py build/tools/conformance
        cp tools/conformance/lcms2.py build/tools/conformance
        cp build/tools/djxl build/tools/conformance
        cp build/lib/libjxl.so.${{ env.LIBJXL_VERSION }} build/tools/conformance
        cp build/lib/libjxl_cms.so.${{ env.LIBJXL_VERSION }} build/tools/conformance
        cp build/lib/libjxl_threads.so.${{ env.LIBJXL_VERSION }} build/tools/conformance
      env:
        SKIP_TEST: 1
    - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2
      with:
        name: conformance_binary-${{ matrix.name }}
        path: |
          build/tools/conformance/conformance.py
          build/tools/conformance/lcms2.py
          build/tools/conformance/djxl
          build/tools/conformance/libjxl.so.${{ env.LIBJXL_VERSION }}
          build/tools/conformance/libjxl_cms.so.${{ env.LIBJXL_VERSION }}
          build/tools/conformance/libjxl_threads.so.${{ env.LIBJXL_VERSION }}
    - name: ccache stats
      run: ccache --show-stats

  run:
    name: Conformance Test ${{ matrix.name }} on ${{ matrix.target }}
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI:none') }}
    needs: [warmup, build]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        name: [main_level5, main_level10]
        target: [AVX3, AVX2, SSE4, SSSE3, EMU128, SCALAR, SCALAR_ASAN]
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
      with:
        egress-policy: audit

    - name: Install deps
      run: |
        pip install numpy
    - name: Checkout the conformance source
      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      with:
        repository: libjxl/conformance
        ref: ${{ env.CONFORMANCE_REPO_HASH }}
        path: conformance
    - name: Cache
      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
      with:
        path: ${{ github.workspace }}/conformance/.objects
        key: conformance-refs
    - name: Download and link conformance files
      run: |
        ${{ github.workspace }}/conformance/scripts/download_and_symlink.sh
    - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4.1.5
      with:
        name: conformance_binary-${{ matrix.target }}
    - name: Run conformance tests
      run: |
        chmod +x djxl
        ln -s libjxl.so.${{ env.LIBJXL_VERSION }} libjxl.so.${{ env.LIBJXL_ABI_VERSION }}
        ln -s libjxl_cms.so.${{ env.LIBJXL_VERSION }} libjxl_cms.so.${{ env.LIBJXL_ABI_VERSION }}
        ln -s libjxl_threads.so.${{ env.LIBJXL_VERSION }} libjxl_threads.so.${{ env.LIBJXL_ABI_VERSION }}
        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`
        python conformance.py \
          --decoder=`pwd`/djxl \
          --corpus=`pwd`/conformance/testcases/${{ matrix.name }}.txt