Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into apachegh-43410-arro…
Browse files Browse the repository at this point in the history
…w-pycapsule-dataset
  • Loading branch information
jorisvandenbossche committed Sep 23, 2024
2 parents 563b30c + 0f7b5e5 commit 0083e02
Show file tree
Hide file tree
Showing 725 changed files with 21,031 additions and 9,525 deletions.
1 change: 1 addition & 0 deletions .clang-format
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ BasedOnStyle: Google
ColumnLimit: 90
DerivePointerAlignment: false
IncludeBlocks: Preserve
IndentPPDirectives: AfterHash
21 changes: 2 additions & 19 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
# include explicitly
!ci/**
!c_glib/Gemfile
!dev/archery/setup.py
!dev/release/setup-*.sh
!docs/requirements*.txt
!go/go.mod
!go/go.sum
!python/requirements*.txt
!python/manylinux1/**
!r/DESCRIPTION
!ruby/Gemfile
!ruby/red-arrow/Gemfile
Expand All @@ -46,20 +46,3 @@
!ruby/red-parquet/Gemfile
!ruby/red-parquet/lib/parquet/version.rb
!ruby/red-parquet/red-parquet.gemspec
!ruby/red-plasma/Gemfile
!ruby/red-plasma/lib/plasma/version.rb
!ruby/red-plasma/red-plasma.gemspec
!rust/Cargo.toml
!rust/benchmarks/Cargo.toml
!rust/arrow/Cargo.toml
!rust/arrow/benches
!rust/arrow-flight/Cargo.toml
!rust/parquet/Cargo.toml
!rust/parquet/build.rs
!rust/parquet_derive/Cargo.toml
!rust/parquet_derive_test/Cargo.toml
!rust/datafusion/Cargo.toml
!rust/datafusion/benches
!rust/integration-testing/Cargo.toml
!go/go.mod
!go/go.sum
8 changes: 4 additions & 4 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,6 @@ CUDA=11.2.2
DASK=latest
DOTNET=8.0
GCC_VERSION=""
GO=1.21.8
STATICCHECK=v0.4.7
HDFS=3.2.1
JDK=11
KARTOTHEK=latest
Expand All @@ -70,7 +68,9 @@ NODE=18
NUMBA=latest
NUMPY=latest
PANDAS=latest
PYTHON=3.8
PYTHON=3.9
PYTHON_IMAGE_TAG=3.9
PYTHON_ABI_TAG=cp39
R=4.4
SPARK=master
TURBODBC=latest
Expand All @@ -95,7 +95,7 @@ VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release
# ci/docker/python-wheel-windows-vs2019.dockerfile.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-06-18
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-08-06

# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker-compose run --rm conan".
# See https://github.com/conan-io/conan-docker-tools#readme and
Expand Down
7 changes: 0 additions & 7 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,6 @@ updates:
commit-message:
prefix: "MINOR: [CI] "
open-pull-requests-limit: 10
- package-ecosystem: "gomod"
directory: "/go/"
schedule:
interval: "weekly"
commit-message:
prefix: "MINOR: [Go] "
open-pull-requests-limit: 10
- package-ecosystem: "maven"
directory: "/java/"
schedule:
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/archery.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ name: Archery & Crossbow
on:
push:
paths:
- '.dockerignore'
- '.github/workflows/archery.yml'
- 'dev/archery/**'
- 'dev/tasks/**'
- 'docker-compose.yml'
pull_request:
paths:
- '.dockerignore'
- '.github/workflows/archery.yml'
- 'dev/archery/**'
- 'dev/tasks/**'
Expand Down Expand Up @@ -58,7 +60,7 @@ jobs:
shell: bash
run: git branch $ARCHERY_DEFAULT_BRANCH origin/$ARCHERY_DEFAULT_BRANCH || true
- name: Setup Python
uses: actions/setup-python@v5.1.1
uses: actions/setup-python@v5.2.0
with:
python-version: '3.9'
- name: Install pygit2 binary wheel
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/comment_bot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
# fetch the tags for version number generation
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: 3.12
- name: Install Archery and Crossbow dependencies
Expand Down
28 changes: 13 additions & 15 deletions .github/workflows/cpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ name: C++
on:
push:
paths:
- '.dockerignore'
- '.github/workflows/cpp.yml'
- 'ci/conda_env_*'
- 'ci/docker/**'
Expand All @@ -35,6 +36,7 @@ on:
- 'testing'
pull_request:
paths:
- '.dockerignore'
- '.github/workflows/cpp.yml'
- 'ci/conda_env_*'
- 'ci/docker/**'
Expand Down Expand Up @@ -99,7 +101,6 @@ jobs:
cat <<JSON >> "$GITHUB_OUTPUT"
{
"arch": "arm64v8",
"archery-use-legacy-docker-compose": "1",
"clang-tools": "10",
"image": "ubuntu-cpp",
"llvm": "10",
Expand All @@ -124,9 +125,6 @@ jobs:
include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
env:
ARCH: ${{ matrix.arch }}
# By default, use `docker compose` because docker-compose v1 is obsolete,
# except where the Docker client version is too old.
ARCHERY_USE_LEGACY_DOCKER_COMPOSE: ${{ matrix.archery-use-legacy-docker-compose || '0' }}
ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
CLANG_TOOLS: ${{ matrix.clang-tools }}
LLVM: ${{ matrix.llvm }}
Expand All @@ -147,6 +145,7 @@ jobs:
run: |
sudo apt update
sudo apt install -y --no-install-recommends python3 python3-dev python3-pip
python3 -m pip install -U pip
- name: Setup Archery
run: python3 -m pip install -e dev/archery[docker]
- name: Execute Docker Build
Expand All @@ -156,8 +155,7 @@ jobs:
run: |
# GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
sudo sysctl -w vm.mmap_rnd_bits=28
sudo sysctl -w kernel.core_pattern="core.%e.%p"
ulimit -c unlimited
source ci/scripts/util_enable_core_dumps.sh
archery docker run ${{ matrix.image }}
- name: Docker Push
if: >-
Expand Down Expand Up @@ -246,7 +244,7 @@ jobs:
$(brew --prefix bash)/bin/bash \
ci/scripts/install_minio.sh latest ${ARROW_HOME}
- name: Set up Python
uses: actions/setup-python@v5.1.1
uses: actions/setup-python@v5.2.0
with:
python-version: 3.12
- name: Install Google Cloud Storage Testbench
Expand All @@ -273,7 +271,7 @@ jobs:
shell: bash
run: |
sudo sysctl -w kern.coredump=1
sudo sysctl -w kern.corefile=core.%N.%P
sudo sysctl -w kern.corefile=/tmp/core.%N.%P
ulimit -c unlimited # must enable within the same shell
ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
Expand Down Expand Up @@ -412,12 +410,10 @@ jobs:
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
# Don't use preinstalled Boost by empty BOOST_ROOT and
# -DBoost_NO_BOOST_CMAKE=ON
# Don't use preinstalled Boost by empty BOOST_ROOT
BOOST_ROOT: ""
ARROW_CMAKE_ARGS: >-
-DARROW_PACKAGE_PREFIX=/${{ matrix.msystem_lower}}
-DBoost_NO_BOOST_CMAKE=ON
-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON
# We can't use unity build because we don't have enough memory on
# GitHub Actions.
Expand Down Expand Up @@ -467,16 +463,18 @@ jobs:
https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z
chmod +x /usr/local/bin/minio.exe
- name: Set up Python
uses: actions/[email protected]
uses: actions/[email protected]
id: python-install
with:
python-version: 3.9
- name: Install Google Cloud Storage Testbench
shell: bash
shell: msys2 {0}
env:
PIPX_BIN_DIR: /usr/local/bin
PIPX_BASE_PYTHON: ${{ steps.python-install.outputs.python-path }}
run: |
ci/scripts/install_gcs_testbench.sh default
echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV
- name: Test
shell: msys2 {0}
run: |
PATH="$(cygpath --unix ${PYTHON_BIN_DIR}):${PATH}"
ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build"
2 changes: 1 addition & 1 deletion .github/workflows/csharp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ jobs:
with:
dotnet-version: ${{ matrix.dotnet }}
- name: Setup Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: 3.12
- name: Checkout Arrow
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
with:
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: 3.12
- name: Install pre-commit
Expand All @@ -66,9 +66,9 @@ jobs:
env:
ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
UBUNTU: 22.04
run: |
sudo sysctl -w kernel.core_pattern="core.%e.%p"
ulimit -c unlimited
source ci/scripts/util_enable_core_dumps.sh
archery docker run -e GITHUB_ACTIONS=true ubuntu-lint
- name: Docker Push
if: >-
Expand Down Expand Up @@ -104,7 +104,7 @@ jobs:
with:
fetch-depth: 0
- name: Install Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: '3.12'
- name: Install Ruby
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
key: debian-docs-${{ hashFiles('cpp/**') }}
restore-keys: debian-docs-
- name: Setup Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: 3.12
- name: Setup Archery
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/docs_light.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ name: Docs
on:
pull_request:
paths:
- '.dockerignore'
- 'docs/**'
- '.github/workflows/docs_light.yml'
- 'ci/docker/conda.dockerfile'
Expand Down Expand Up @@ -58,7 +59,7 @@ jobs:
key: conda-docs-${{ hashFiles('cpp/**') }}
restore-keys: conda-docs-
- name: Setup Python
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: 3.12
- name: Setup Archery
Expand Down
Loading

0 comments on commit 0083e02

Please sign in to comment.