Skip to content

Commit

Permalink
feat: Polars build for Pyodide (#20383)
Browse files Browse the repository at this point in the history
Co-authored-by: ritchie <[email protected]>
  • Loading branch information
georgestagg and ritchie46 authored Dec 20, 2024
1 parent e074e1a commit 530e70a
Show file tree
Hide file tree
Showing 29 changed files with 393 additions and 198 deletions.
74 changes: 72 additions & 2 deletions .github/workflows/release-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,70 @@ jobs:
name: wheel-${{ matrix.package }}-${{ matrix.os }}-${{ matrix.architecture }}
path: dist/*.whl

build-wheel-pyodide:
name: build-wheels (polars, pyodide, wasm32)
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.sha }}

# Avoid potential out-of-memory errors
- name: Set swap space for Linux
uses: pierotofy/set-swap-space@master
with:
swap-size-gb: 10

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Disable incompatible features
env:
FEATURES: parquet|async|json|extract_jsonpath|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
run: |
sed -i 's/^ "json",$/ "serde_json",/' crates/polars-python/Cargo.toml
sed -E -i "/^ \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/Cargo.toml
- name: Setup emsdk
uses: mymindstorm/setup-emsdk@v14
with:
# This should match the exact version of Emscripten used by Pyodide
version: 3.1.58

- name: Install LLVM
# This should match the major version of LLVM expected by Emscripten
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 19
echo "EM_LLVM_ROOT=/usr/lib/llvm-19/bin" >> $GITHUB_ENV
- name: Set CFLAGS and RUSTFLAGS for wasm32
run: |
echo "CFLAGS=-fPIC" >> $GITHUB_ENV
echo "RUSTFLAGS=-C link-self-contained=no" >> $GITHUB_ENV
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
command: build
target: wasm32-unknown-emscripten
args: >
--profile dist-release
--manifest-path py-polars/Cargo.toml
--interpreter python3.10
--out wasm-dist
maturin-version: 1.7.4

- name: Upload wheel
uses: actions/upload-artifact@v4
with:
name: wheel-polars-emscripten-wasm32
path: wasm-dist/*.whl

publish-to-pypi:
needs: [create-sdist, build-wheels]
environment:
Expand All @@ -327,7 +391,7 @@ jobs:
verbose: true

publish-to-github:
needs: publish-to-pypi
needs: [publish-to-pypi, build-wheel-pyodide]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand All @@ -340,6 +404,12 @@ jobs:
name: sdist-polars
path: dist

- name: Download Pyodide wheel
uses: actions/download-artifact@v4
with:
name: wheel-polars-emscripten-wasm32
path: wasm-dist

- name: Get version from Cargo.toml
id: version
working-directory: py-polars
Expand Down Expand Up @@ -372,7 +442,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ steps.github-release.outputs.tag_name }}
FILES: dist/polars-*.tar.gz
FILES: dist/polars-*.tar.gz wasm-dist/polars-*.whl

- name: Publish GitHub release
if: inputs.dry-run == false
Expand Down
70 changes: 70 additions & 0 deletions .github/workflows/test-pyodide.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: Test Pyodide

on:
pull_request:
paths:
- Cargo.lock
- py-polars/**
- docs/source/src/python/**
- crates/**
- .github/workflows/test-pyodide.yml
push:
branches:
- main
paths:
- Cargo.lock
- crates/**
- docs/source/src/python/**
- py-polars/**
- .github/workflows/test-pyodide.yml

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
RUSTFLAGS: -C link-self-contained=no -C debuginfo=0
CFLAGS: -fPIC

defaults:
run:
shell: bash

jobs:
test-pyodide:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Disable incompatible features
env:
FEATURES: parquet|async|json|extract_jsonpath|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
run: |
sed -i 's/^ "json",$/ "serde_json",/' crates/polars-python/Cargo.toml
sed -E -i "/^ \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/Cargo.toml
- name: Setup emsdk
uses: mymindstorm/setup-emsdk@v14
with:
# This should match the exact version of Emscripten used by Pyodide
version: 3.1.58

- name: Install LLVM
# This should match the major version of LLVM expected by Emscripten
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 19
echo "EM_LLVM_ROOT=/usr/lib/llvm-19/bin" >> $GITHUB_ENV
- name: Build wheel
uses: PyO3/maturin-action@v1
with:
command: build
target: wasm32-unknown-emscripten
args: >
--profile dev
--manifest-path py-polars/Cargo.toml
--interpreter python3.10
maturin-version: 1.7.4
Loading

0 comments on commit 530e70a

Please sign in to comment.