chore: CI Refactor #3306
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration | |
| on: # rebuild any PRs and main branch changes | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - 'releases/*' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # FIXME: Find a better name | |
| IS_RELEASE_BRANCH: | | |
| ${{ | |
| (github.ref == 'refs/heads/main' | |
| || startsWith(github.ref, 'refs/tags/') | |
| || startsWith(github.ref, 'refs/heads/releases')) | |
| && github.event_name != 'pull_request' | |
| }} | |
| # NODE_OPTIONS: --max_old_space_size=4096 | |
| jobs: | |
| # Compile native bridge code and isolate extension for Windows, Mac and Linux. | |
| # Uploads the packages as a build artifact to be tested later. | |
| compile-binaries: | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| # Use Buildjet's Unbuntu 18.04 builders to preserve compatibility with glibc 2.17 | |
| - os: ubuntu | |
| runner: buildjet-2vcpu-ubuntu-1804 | |
| target: x86_64-unknown-linux-gnu | |
| - os: ubuntu | |
| runner: buildjet-2vcpu-ubuntu-1804 | |
| target: aarch64-unknown-linux-gnu | |
| rustflags: '-C linker=aarch64-linux-gnu-gcc' | |
| - os: macos | |
| runner: macos-latest | |
| target: x86_64-apple-darwin | |
| - os: macos | |
| runner: macos-latest | |
| target: aarch64-apple-darwin | |
| - os: windows | |
| runner: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| runs-on: ${{ matrix.runner }} | |
| container: ${{ matrix.container }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: 'Checkout code' | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: 'Cache index.node' | |
| id: cached-artifact | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./packages/core-bridge/releases | |
| key: corebridge-artifactcache-${{ matrix.target }}-${{ hashFiles('./packages/core-bridge/**/Cargo.lock', './packages/core-bridge/**/*.rs') }} | |
| - name: Install Rust | |
| if: steps.cached-artifact.outputs.cache-hit != 'true' | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| target: ${{ matrix.target }} | |
| - name: Install protoc | |
| if: steps.cached-artifact.outputs.cache-hit != 'true' | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| version: '3.x' | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Print libc version | |
| if: (steps.cached-artifact.outputs.cache-hit != 'true') && (matrix.os == 'ubuntu') | |
| run: ldd --version | |
| - name: Install gcc-aarch64-linux-gnu | |
| if: (steps.cached-artifact.outputs.cache-hit != 'true') && (matrix.target == 'aarch64-unknown-linux-gnu') | |
| run: sudo apt update && sudo apt install -y gcc-aarch64-linux-gnu | |
| - name: Rust Cargo and Build cache | |
| if: steps.cached-artifact.outputs.cache-hit != 'true' | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: packages/core-bridge -> target | |
| prefix-key: corebridge-buildcache | |
| shared-key: ${{ matrix.target }} | |
| env-vars: '' | |
| # save-if: (github.ref == 'refs/heads/main') | |
| - name: Cross compile rust code | |
| if: steps.cached-artifact.outputs.cache-hit != 'true' | |
| env: | |
| RUSTFLAGS: ${{ matrix.rustflags }} | |
| working-directory: ./packages/core-bridge | |
| run: | | |
| cargo build --release --target ${{ matrix.target }} | |
| find ./target/${{ matrix.target }}/ -ls | |
| mkdir -p ./releases/${{ matrix.target }} | |
| for i in ./target/${{ matrix.target }}/release/{libtemporal_sdk_typescript_bridge.{dylib,so},temporal_sdk_typescript_bridge.dll} ; do | |
| ls [[ -x $i ]] && -l matrix.target }} | |
| cp -a $i ./releases/${{ matrix.target }}/index.node || true | |
| done | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: corebridge-native-${{ matrix.target }} | |
| # Actual file will be named ${{ matrix.target }}/index.node | |
| path: ./packages/core-bridge/releases/*/index.node | |
| build-packages: | |
| needs: | |
| - compile-binaries | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Download core-bridge native libraries | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ./packages/core-bridge/releases/tmp | |
| - name: Put native files into place | |
| working-directory: ./packages/core-bridge/releases | |
| run: | | |
| mv tmp/corebridge-*/* ./ | |
| rm -rf tmp | |
| - name: Install Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Get NPM cache directory | |
| id: npm-cache-dir | |
| run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| - name: Restore NPM cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-ubuntu-${{ hashFiles('./package-lock.json') }} | |
| restore-keys: | | |
| npm-main-ubuntu- | |
| - name: Download dependencies | |
| run: | | |
| npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose | |
| - name: Compile code | |
| run: npm run build -- --ignore @temporalio/core-bridge | |
| - name: Publish to Verdaccio | |
| run: node scripts/publish-to-verdaccio.js --registry-dir ./tmp/registry | |
| - name: Save Verdaccio repo artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: verdaccio-repo | |
| path: ./tmp/registry/storage | |
| - name: Save NPM cache | |
| uses: actions/cache/save@v3 | |
| # Only saves NPM cache from the main branch, to reduce pressure on the cache (limited to 10GB). | |
| # if: github.ref == 'refs/heads/main' # FIXME | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-ubuntu-${{ hashFiles('./package-lock.json') }} | |
| integration-tests: | |
| needs: | |
| - compile-binaries | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: [14, 16, 18, 20] | |
| os: [ubuntu, macos, windows] | |
| reuse-v8-context: [true, false] | |
| server: [cli] # FIXME: Add 'cloud', 'testenv' | |
| # exclude: | |
| # FIXME: Disable 'cloud' if 'TEMPORAL_CLIENT_CERT' isn't set | |
| # - server: "${{ env.TEMPORAL_CLIENT_CERT == '' && 'cloud' || 'ignore' }}" | |
| include: | |
| - os: ubuntu | |
| runner: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - os: macos | |
| runner: macos-latest | |
| target: x86_64-apple-darwin | |
| - os: windows | |
| runner: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| runs-on: ${{ matrix.runner }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Print build information | |
| run: 'echo head_ref: ${{ github.head_ref }}, ref: ${{ github.ref }}, os: ${{ matrix.os }}, node: ${{ matrix.node }}' | |
| - name: 'Checkout code' | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Download core-bridge native libraries | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: corebridge-native-${{ matrix.target }} | |
| path: ./packages/core-bridge/releases | |
| - name: Install Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node }} | |
| - name: Get NPM cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| - name: Restore NPM cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-${{ runner.os }}-${{ hashFiles('./package-lock.json') }} | |
| restore-keys: | | |
| npm-main-${{ runner.os }}- | |
| - name: Download dependencies | |
| run: | | |
| npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose | |
| - name: Compile code | |
| run: npm run build -- --ignore @temporalio/core-bridge | |
| - name: Install Temporal CLI | |
| if: ${{ matrix.server == 'cli' }} | |
| uses: temporalio/setup-temporal@v0 | |
| - name: Runs Temporal CLI | |
| if: ${{ matrix.server == 'cli' }} | |
| shell: bash | |
| run: | | |
| temporal server start-dev --headless & | |
| - name: Print out Node heap statistics | |
| run: node -e 'console.log(v8.getHeapStatistics())' | |
| - name: Run Tests | |
| run: npm test | |
| env: | |
| RUN_INTEGRATION_TESTS: ${{ matrix.server == 'cli' }} | |
| REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }} | |
| - uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: integration-tests-${{ matrix.os }}-node${{ matrix.node }}-${{ matrix.server }}-logs | |
| path: | | |
| ~/.npm/_logs/ | |
| C:\npm\cache\_logs\ | |
| # Tests that npm init @temporalio results in a working worker and client | |
| test-npm-init: | |
| needs: build-packages | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: [14, 16, 18] | |
| os: [ubuntu, macos, windows] | |
| sample: [hello-world, fetch-esm, hello-world-mtls] | |
| server: [cli, cloud] | |
| exclude: | |
| # Disable non-mtls tests on cloud | |
| - sample: hello-world | |
| server: cloud | |
| # Disable mtls tests on cli | |
| - sample: hello-world-mtls | |
| server: cli | |
| - sample: fetch-esm | |
| server: cloud | |
| # FIXME: investigate why 'fetch-esm' always hangs on Windows | |
| - sample: fetch-esm | |
| os: windows | |
| # FIXME: Disable 'cloud' if 'TEMPORAL_CLIENT_CERT' isn't set | |
| # - server: "${{ env.TEMPORAL_CLIENT_CERT == '' && 'cloud' || 'ignore' }}" | |
| include: | |
| - os: ubuntu | |
| runner: ubuntu-latest | |
| - os: macos | |
| runner: macos-latest | |
| - os: windows | |
| runner: windows-latest | |
| runs-on: ${{ matrix.runner }} | |
| env: | |
| TEMPORAL_CLIENT_CERT: ${{ secrets.TEMPORAL_CLIENT_CERT }} | |
| TEMPORAL_CLIENT_KEY: ${{ secrets.TEMPORAL_CLIENT_KEY }} | |
| steps: | |
| - name: 'Checkout code' | |
| uses: actions/checkout@v3 | |
| with: | |
| # We don't need the core submodule here since won't build the project | |
| submodules: false | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Install Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.node }} | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Get NPM cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Restore NPM cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-${{ runner.os }}-${{ hashFiles('./package-lock.json') }} | |
| restore-keys: | | |
| npm-main-${{ runner.os }}- | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| # No need to compile anything, we just need the package ./scripts and their dependencies | |
| - name: Install dependencies without compilation | |
| run: | | |
| npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Restore Verdaccio repo artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: verdaccio-repo | |
| path: ./tmp/registry/storage | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| - run: find . -ls | |
| shell: bash | |
| working-directory: ./tmp/registry/storage | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| # Note: here, `npx create` fails on windows if shell is bash. | |
| - name: Instantiate sample project using verdaccio artifacts | |
| run: node scripts/init-from-verdaccio.js --registry-dir ./tmp/registry --sample https://github.com/temporalio/samples-typescript/tree/next/${{ matrix.sample }} --target-dir ${{ runner.temp }}/example | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| # FIXME: Move this to an independant github action | |
| - name: Install Temporal CLI | |
| if: ${{ matrix.server == 'cli' }} | |
| uses: temporalio/setup-temporal@v0 | |
| - name: Runs Temporal CLI | |
| if: ${{ matrix.server == 'cli' }} | |
| shell: bash | |
| run: | | |
| temporal server start-dev --headless & | |
| - name: Print out Node heap statistics | |
| run: node -e 'console.log(v8.getHeapStatistics()) ; console.log(v8.getHeapStatistics())' | |
| working-directory: ${{ runner.temp }}/example | |
| if: ${{ matrix.server != 'cloud' || env.TEMPORAL_CLIENT_CERT != '' }} | |
| # We write the certs to disk because it serves the sample. Written into /tmp/temporal-certs | |
| - name: Create certs dir | |
| shell: bash | |
| run: node scripts/create-certs-dir.js "${{ runner.temp }}/certs" | |
| if: ${{ matrix.server == 'cloud' && env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Test run a workflow (non-cloud) | |
| run: node scripts/test-example.js --work-dir "${{ runner.temp }}/example" | |
| shell: bash | |
| env: | |
| TEMPORAL_ADDRESS: localhost | |
| TEMPORAL_NAMESPACE: default | |
| TEMPORAL_TASK_QUEUE: ${{ format('{0}-{1}-{2}', matrix.os, matrix.node, matrix.target) }} | |
| if: ${{ matrix.server == 'cli' }} | |
| - name: Test run a workflow (cloud) | |
| run: node scripts/test-example.js --work-dir "${{ runner.temp }}/example" | |
| shell: bash | |
| env: | |
| # TODO: get a permanent cloud namespace for CI | |
| # These env vars are ignored by the docker server example | |
| TEMPORAL_ADDRESS: sdk-ci.a2dd6.tmprl.cloud | |
| TEMPORAL_NAMESPACE: sdk-ci.a2dd6 | |
| TEMPORAL_CLIENT_CERT_PATH: ${{ runner.temp }}/certs/client.pem | |
| TEMPORAL_CLIENT_KEY_PATH: ${{ runner.temp }}/certs/client.key | |
| TEMPORAL_TASK_QUEUE: ${{ format('{0}-{1}-{2}', matrix.os, matrix.node, matrix.target) }} | |
| if: ${{ matrix.server == 'cloud' && env.TEMPORAL_CLIENT_CERT != '' }} | |
| - name: Destroy certs dir | |
| run: rm -rf ${{ runner.temp }}/certs | |
| shell: bash | |
| if: ${{ matrix.server == 'cloud' && env.TEMPORAL_CLIENT_CERT != '' }} | |
| # Runs the features repo tests with this repo's current SDK code | |
| features-tests: | |
| uses: temporalio/features/.github/workflows/typescript.yaml@main | |
| with: | |
| typescript-repo-path: ${{github.event.pull_request.head.repo.full_name}} | |
| version: ${{github.event.pull_request.head.ref}} | |
| version-is-repo-ref: true | |
| stress-tests: | |
| uses: ./.github/workflows/stress.yml | |
| with: | |
| test-type: ci-stress | |
| test-timeout-minutes: 20 | |
| reuse-v8-context: false | |
| stress-tests-reuse-context: | |
| uses: ./.github/workflows/stress.yml | |
| with: | |
| test-type: ci-stress | |
| test-timeout-minutes: 20 | |
| reuse-v8-context: true | |
| # Run TS linting and ts-prune to find unused code | |
| lint-and-prune: | |
| strategy: | |
| # Using a matrix here ensure that Rust-related actions below can be easily be copied from the | |
| # compile-binairies job and that the Rust build cache will be usable | |
| matrix: | |
| include: | |
| - os: ubuntu | |
| runner: ubuntu-22.04 | |
| target: x86_64-unknown-linux-gnu | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Get NPM cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| - name: Restore NPM cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-${{ runner.os }}-${{ hashFiles('./package-lock.json') }} | |
| restore-keys: | | |
| npm-main-${{ runner.os }}- | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| target: ${{ matrix.target }} | |
| - name: Install protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| version: '3.x' | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Rust Cargo and Build cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: packages/core-bridge -> target | |
| prefix-key: corebridge-buildcache | |
| shared-key: ${{ matrix.target }} | |
| env-vars: '' | |
| save-if: false | |
| - name: Download dependencies | |
| run: | | |
| npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose | |
| # eslint-import-resolver-typescript requires packages to be built | |
| - name: Compile all non-rust code | |
| run: npm run build -- --ignore @temporalio/core-bridge | |
| - run: npm run lint.check | |
| - run: npm run lint.prune | |
| build-docs: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16 | |
| - name: Get NPM cache directory | |
| id: npm-cache-dir | |
| shell: bash | |
| run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| - name: Restore NPM cache | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: ${{ steps.npm-cache-dir.outputs.dir }} | |
| key: npm-main-${{ runner.os }}-${{ hashFiles('./package-lock.json') }} | |
| restore-keys: | | |
| npm-main-${{ runner.os }}- | |
| # Don't build during install phase since we're going to explicitly build | |
| - name: Download dependencies | |
| run: | | |
| npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose || npm ci --ignore-scripts --verbose | |
| - run: npm run build -- --ignore @temporalio/core-bridge | |
| # Do docs stuff (only on one host) | |
| - name: Build docs | |
| run: npm run docs | |
| env: | |
| ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} | |
| - name: Deploy prod docs # TODO: only deploy prod docs when we publish a new version | |
| if: ${{ github.ref == 'refs/heads/main' }} | |
| run: npx vercel deploy packages/docs/build -t ${{ secrets.VERCEL_TOKEN }} --name typescript --scope temporal --prod --yes | |
| - name: Deploy draft docs | |
| # Don't run on forks, since secrets won't be available, and command will fail | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.ref != 'refs/heads/main' }} | |
| run: npx vercel deploy packages/docs/build -t ${{ secrets.VERCEL_TOKEN }} --name typescript --scope temporal --yes |