[feature] Support remote and compressed index files #1094
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: '**' | |
| tags-ignore: '**' | |
| pull_request: | |
| jobs: | |
| Static-Code-Checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install pip Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --user fusepy pytest lz4 PySquashfsImage asyncssh fsspec | |
| - name: Style Check With Black | |
| run: | | |
| python3 -m pip install black | |
| black -q --diff --line-length 120 --skip-string-normalization ratarmount.py tests/*.py core/ratarmountcore/*.py > black.diff | |
| if [ -s black.diff ]; then | |
| cat black.diff | |
| exit 123 | |
| fi | |
| - name: Lint With Codespell | |
| run: | | |
| python3 -m pip install codespell | |
| codespell $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' ) | |
| - name: Lint With Flake8 | |
| run: | | |
| python3 -m pip install flake8 | |
| flake8 --config tests/.flake8 *.py tests/*.py core/ratarmountcore/[^_]*.py | |
| - name: Lint With Pylint | |
| run: | | |
| python3 -m pip install pylint | |
| pylint --rcfile tests/.pylintrc *.py tests/*.py core/ratarmountcore/*.py | tee pylint.log | |
| ! 'egrep' ': E[0-9]{4}: ' pylint.log | |
| - name: Lint With Pytype | |
| run: | | |
| python3 -m pip install pytype | |
| pytype -d import-error -P$( cd core && pwd ):$( pwd ) ratarmount.py core/ratarmountcore/*.py | |
| - name: Lint With Mypy | |
| run: | | |
| yes | python3 -m pip install --upgrade-strategy eager --upgrade types-dataclasses mypy | |
| mypy --config-file tests/.mypy.ini *.py core/ratarmountcore/*.py | |
| yes | python3 -m pip uninstall types-dataclasses | |
| - name: Lint With ShellCheck | |
| run: | | |
| sudo apt-get -y install shellcheck | |
| shellcheck -e SC2064 tests/*.sh | |
| Tests: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: ['macos-13', 'ubuntu-latest'] | |
| # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels, | |
| # probably because it is M1 based. | |
| # ToDo: Add windows-latest but it requires a lot of setup of the dependencies! | |
| # Maybe only test ratarmount-core without most dependencies after I have split that off. | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # https://endoflife.date/python | |
| # 3.5 was released 2015-09-13 and end-of-life was 2020-09-13 | |
| # 3.6 was released 2016-12-23 and end-of-life was 2021-12-23 | |
| # 3.7 was released 2018-06-27 and end-of-life was 2023-06-27 | |
| # 3.8 was released 2019-10-14 and end-of-life will be 2024-10-14 | |
| # 3.9 was released 2020-10-05 and end-of-life will be 2025-10-25 | |
| # 3.10 was released 2021-10-04 and end-of-life will be 2026-10-04 | |
| # 3.11 was released 2022-10-24 and end-of-life will be 2027-10 | |
| # 3.12 was released 2023-10-02 and end-of-life will be 2028-10 | |
| # 3.13 is to be released 2024-10 | |
| # 3.14 is to be released 2025-10 | |
| python-version: ['3.9', '3.12', '3.13.0'] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: '3.14.0-alpha.0' | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # We need one tag for testing the git mount. | |
| # This is BROKEN! God damn it. Is anything working at all... | |
| # https://github.com/actions/checkout/issues/1781 | |
| fetch-tags: true | |
| - name: Fetch tag for tests | |
| run: git fetch origin refs/tags/v0.15.2:refs/tags/v0.15.2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Print System Information | |
| run: | | |
| echo "uname -a: $( uname -a )" | |
| echo "Shell: $SHELL" | |
| echo "Cores: $( nproc )" | |
| echo "Mount points:"; mount | |
| - uses: msys2/setup-msys2@v2 | |
| if: startsWith( matrix.os, 'windows' ) | |
| with: | |
| install: gcc make liblzma-devel libzstd-devel zlib-devel | |
| - name: Install Dependencies (Linux) | |
| if: startsWith( matrix.os, 'ubuntu' ) | |
| run: | | |
| # Libarchive calls the grzip, lrzip, lzop binaries for lrzip support. Others, such as bzip2, gzip, lz4, lzma, | |
| # zstd, may also call external binaries depending on how libarchive was compiled! | |
| # https://github.com/libarchive/libarchive/blob/ad5a0b542c027883d7069f6844045e6788c7d70c/libarchive/ | |
| # archive_read_support_filter_lrzip.c#L68 | |
| sudo apt-get -y install libfuse2 fuse3 bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev ruby-webrick | |
| - name: Install Dependencies For Unreleased Python Versions (Linux) | |
| if: > | |
| startsWith( matrix.os, 'ubuntu' ) && ( | |
| matrix.python-version == '3.13.0' || | |
| matrix.python-version == '3.14.0-alpha.0') | |
| run: | | |
| #libgit2-dev is too old on Ubuntu 22.04. Leads to error about missing git2/sys/errors.h | |
| #sudo apt-get -y install libgit2-dev | |
| sudo apt-get -y install cmake | |
| git clone --branch v1.8.1 --depth 1 https://github.com/libgit2/libgit2.git | |
| ( cd libgit2 && mkdir build && cd build && cmake .. && cmake --build . && sudo cmake --build . -- install ) | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> "$GITHUB_ENV" | |
| # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies. | |
| python3 -m pip install pygit2==1.15 | |
| - name: Install Dependencies (MacOS) | |
| if: startsWith( matrix.os, 'macos' ) | |
| run: | | |
| # coreutils is required for the tests written in shell, e.g., for the 'realpath' command | |
| # unar is required for rar tests with passwords. By default, bsdtar is installed but that is the only | |
| # one of the three supported tools (the third is unrar) that does not support passwords. | |
| # And the error message is atrocious: | |
| # cmdline.extend(args) | |
| # TypeError: 'NoneType' object is not iterable | |
| brew install macfuse coreutils pixz pbzip2 zstd unar libarchive lrzip lzop lzo | |
| # Add brew installation binary folder to PATH so that command line tools like zstd can be found | |
| echo PATH="$PATH:/usr/local/bin" >> "$GITHUB_ENV" | |
| - name: Install Dependencies For Unreleased Python Versions (MacOS) | |
| if: > | |
| startsWith( matrix.os, 'macos' ) && ( | |
| matrix.python-version == '3.13.0' || | |
| matrix.python-version == '3.14.0-alpha.0') | |
| run: | | |
| brew install [email protected] | |
| brew link [email protected] --force | |
| # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies. | |
| python3 -m pip install pygit2==1.15 | |
| - name: Install pip Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade wheel | |
| python3 -m pip install --upgrade setuptools | |
| python3 -m pip install --upgrade-strategy eager --upgrade twine build fusepy cffi | |
| - name: Test Startup With Only One Compression Dependency | |
| run: | | |
| for module in indexed_gzip indexed_zstd lzmaffi python-xz; do | |
| python3 -m pip install --upgrade "$module" | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| # indexed_zstd=1.2.0 did segfault here! | |
| python3 ratarmount.py README.md || [ $? != 139 ] | |
| python3 -m pip uninstall --yes "$module" | |
| done | |
| python3 -m pip install --upgrade 'git+https://github.com/mxmlnkn/indexed_bzip2.git@master#egginfo=rapidgzip&subdirectory=python/rapidgzip' | |
| - name: Test ratarmountcore Installation From Tarball | |
| working-directory: core | |
| run: | | |
| python3 -m build | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Installation From Tarball | |
| run: | | |
| python3 -m build | |
| # Around commit 08ddc8fb3679faacd6e54eb5c54104995b045dc4, I had some very weird twine check failures | |
| # because .tar.gz tarballs were wrongly recognized as ZIP files by CPython's zipfile.is_zipfile. | |
| # Therefore, try recompressing the tarballs with different compression levels until it does not randomly | |
| # look like a ZIP anymore. It is an ugly hack, but so is zipfile.is_zipfile. | |
| tarball=$( find dist -name '*.tar.gz' ) | |
| if python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball"; then | |
| gzip -c -d "$tarball" > "$tarball.tar" | |
| for (( i=9; i>0; --i )); do | |
| cat "$tarball.tar" | gzip -$i > "$tarball" | |
| if ! python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball" | |
| then break; fi | |
| done | |
| fi | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Installation From Source | |
| run: | | |
| python3 -m pip install .[full] | |
| - name: Test Simple Startup | |
| run: | | |
| ratarmount --help | |
| ratarmount --version | |
| - name: Test Simple Mount | |
| # macOS 11+ is too uptight about "security" and is not able to fully load the macfuse kernel extension. | |
| # https://github.com/actions/runner-images/issues/4731 | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| ratarmount tests/single-file.tar mimi | |
| ls -la mimi | |
| sleep 1s | |
| # MacOS does not have fusermount! | |
| ratarmount -u mimi | |
| - name: Test Startup Without Compression Dependencies | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| python3 ratarmount.py tests/simple.bz2 || [ $? != 139 ] | |
| - name: Unit Tests | |
| run: | | |
| python3 -m pip install pytest pytest-xdist pandas | |
| for file in core/tests/test_*.py tests/test_*.py; do | |
| case "$file" in | |
| "tests/test_cli.py") | |
| if ! uname | 'grep' -q -i darwin; then | |
| # First off, n=auto seems to use the physical cores and ignores virtual ones. | |
| # Secondly, these tests scale much better than the others because most time is spent waiting for | |
| # the FUSE mount point to appear or disappear, which doesn't seem to be bottlenecked by CPU usage. | |
| python3 -X dev -W ignore::DeprecationWarning:fuse -u \ | |
| -c "import pytest, re, sys; sys.exit(pytest.console_main())" \ | |
| -n 16 --disable-warnings "$testFile" | |
| fi | |
| ;; | |
| "core/tests/test_AutoMountLayer.py"\ | |
| |"core/tests/test_BlockParallelReaders.py"\ | |
| |"core/tests/test_LibarchiveMountSource.py"\ | |
| |"core/tests/test_SQLiteIndexedTar.py") | |
| echo "$file" # pytest-xdist seems to omit the test file name | |
| pytest -n auto --disable-warnings "$file" | |
| ;; | |
| *) | |
| # Fusepy warns about usage of use_ns because the implicit behavior is deprecated. | |
| # But there has been no development to fusepy for 4 years, so I think it should be fine to ignore. | |
| pytest --disable-warnings "$file" | |
| esac | |
| done | |
| python3 tests/tests.py | |
| - name: Install pip Test Dependencies | |
| run: | | |
| python3 -m pip install -r tests/requirements-tests.txt | |
| - name: Install Test Dependencies (Linux) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| sudo bash tests/install-smbd.sh | |
| - name: Regression Tests (FUSE 3) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| run: | | |
| export FUSE_LIBRARY_PATH=$( dpkg -L libfuse3-3 | 'grep' -F .so | head -1 ) | |
| ratarmount --version | |
| bash tests/runtests.sh | |
| - name: Regression Tests (FUSE 2) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| run: | | |
| bash tests/runtests.sh | |
| python3 -c 'import pygit2' | |
| - name: Module tests without fusepy | |
| run: | | |
| python3 -m pip uninstall -y fusepy | |
| python3 tests/tests.py |