Run gprofiler without root/sudo #464
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # build-executable-{arch} job builds the gprofiler executable by running build_{arch}_executable.sh scripts, then uploads it as a job artifact | |
| # test-executable-{arch} job downloads gprofiler exe artifact and then runs test_executable.py tests from tests dir | |
| # deploy-executable job runs only on tagged commits and it deploys gprofiler executables as release assets | |
| # build-container-{arch} job downloads gprofiler exe artifact, produces gprofiler docker image from it, exports it into the file, and uploads image as a job artifact | |
| # test-container-{arch} job downloads gprofiler image artifact and runs tests/test.sh script | |
| # deploy-container job runs only on tagged commits and it deploys gprofiler images to dockerhub | |
| name: Build, test and deploy | |
| on: | |
| pull_request: | |
| push: | |
| tags: | |
| - '**' | |
| jobs: | |
| build-executable-x64: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
| - name: Build gProfiler executable | |
| # Using BuildKit although it has another cache mechanism which is not supported by satackey/[email protected] | |
| # We tried to cache using buildx cache (cache-from, cache-to flags) and got strange behavior when caching, | |
| # decided not to use buildkit here to get nice caches. | |
| run: | | |
| mkdir -p output | |
| ./scripts/build_x86_64_executable.sh | |
| mv build/x86_64/gprofiler output/gprofiler_x86_64 | |
| cp output/gprofiler_x86_64 output/gprofiler # for backwards compatibility, we upload both with arch suffix and without | |
| - name: Upload the executables as job artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64 | |
| path: output/ | |
| retention-days: 1 | |
| test-executable-x64: | |
| if: ${{ !startsWith(github.ref, 'refs/tags/') }} | |
| runs-on: ubuntu-20.04 # the tests which use ruby/node/python etc and run at non-root privs, fail to read the files when running | |
| # on ubuntu-22.04/ubuntu-latest: | |
| # stderr: ruby: Permission denied -- /home/runner/work/gprofiler/gprofiler/tests/containers/ruby/fibonacci.rb (LoadError) | |
| needs: build-executable-x64 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| containers: | |
| - alpine | |
| - ubuntu:14.04 | |
| - ubuntu:16.04 | |
| - ubuntu:18.04 | |
| - ubuntu:20.04 | |
| - ubuntu:20.10 | |
| - ubuntu:22.04 | |
| - centos:6 | |
| - centos:7 | |
| - centos:8 | |
| - debian:8 | |
| - debian:9 | |
| - debian:10 | |
| - debian:11 | |
| steps: | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8" | |
| - name: Install Java | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8.0.275' | |
| java-package: jdk | |
| architecture: x64 | |
| - name: Install Node.JS | |
| uses: actions/setup-node@v2 | |
| with: | |
| # same version as used in tests/containers/nodejs/Dockerfile | |
| node-version: 10.x | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Download the executable from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64 | |
| path: dist/ | |
| - name: Run gProfiler tests | |
| run: | | |
| mv dist/gprofiler_x86_64 dist/gprofiler | |
| chmod +x dist/gprofiler | |
| NO_APT_INSTALL=1 ./tests/test.sh --exec-container-image ${{ matrix.containers }} --executable dist/gprofiler -k test_executable | |
| build-executable-aarch64: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: | |
| - self-hosted | |
| - public | |
| - ARM64 | |
| - gprofiler | |
| - EC2 | |
| - linux | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Set up requirements | |
| run: scripts/setup_runner_requirements.sh | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build gProfiler executable | |
| run: | | |
| mkdir -p output | |
| ./scripts/build_aarch64_executable.sh | |
| mv build/aarch64/gprofiler output/gprofiler_aarch64 | |
| - name: Upload the executables as job artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gprofiler_aarch64 | |
| path: output/ | |
| retention-days: 1 | |
| deploy-executable: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - build-executable-x64 | |
| - build-executable-aarch64 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| # verification before we deploy | |
| - name: Verify release tag matches gProfiler version | |
| run: ./scripts/verify_tag.sh | |
| - name: Download x86_64 executable from a previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64 | |
| path: output/ | |
| - name: Download aarch64 executable from a previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_aarch64 | |
| path: output/ | |
| - name: Release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: | | |
| output/gprofiler_x86_64 | |
| output/gprofiler_aarch64 | |
| output/gprofiler | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| build-container-x64: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-executable-x64 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| fetch-depth: 0 | |
| - name: Download executables from the previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64 | |
| path: output/ | |
| # the tests need the gprofiler image built (from Dockerfile). I run it separately here, because "docker build" prints the build logs | |
| # more nicely. the tests will then be able to use the built image. | |
| - name: Build gProfiler image | |
| # see https://github.com/docker/buildx/issues/1507 about the --provenance flag, I decided to go safe without the extra manifest. | |
| # --skip-exe-build needs to be first! | |
| run: ./scripts/build_x86_64_container.sh --skip-exe-build --provenance=false --build-arg EXE_PATH=output/gprofiler_x86_64 -t gprofiler_x86_64 | |
| - name: Export gProfiler image | |
| run: mkdir -p output && docker image save gprofiler_x86_64 > output/gprofiler_x86_64.img | |
| - name: Upload the image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64.img | |
| path: output/ | |
| retention-days: 1 | |
| test-container-x64: | |
| if: ${{ !startsWith(github.ref, 'refs/tags/') }} | |
| needs: build-container-x64 | |
| runs-on: ubuntu-20.04 # for 3.10.0 python tag | |
| strategy: | |
| fail-fast: false # helps detecting flakiness / errors specific to one Python version | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10.0" # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests. | |
| steps: | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Java | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: '8.0.275' | |
| java-package: jdk | |
| architecture: x64 | |
| - name: Install Node.JS | |
| uses: actions/setup-node@v2 | |
| with: | |
| # same version as used in tests/containers/nodejs/Dockerfile | |
| node-version: 10.x | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Download the executable from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64 | |
| path: output/ | |
| - name: Download the image from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64.img | |
| path: output/ | |
| - name: Add +x to gprofiler | |
| run: chmod +x ./output/gprofiler_x86_64 | |
| - name: Extract resources from gProfiler executable | |
| run: sudo ./output/gprofiler_x86_64 extract-resources --resources-dest=./gprofiler/resources | |
| # used in the tests | |
| - name: Import gProfiler image | |
| run: docker image load < output/gprofiler_x86_64.img | |
| # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
| - name: Run gProfiler tests | |
| run: NO_APT_INSTALL=1 ./tests/test.sh --ignore=tests/test_executable.py | |
| build-container-aarch64: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| needs: build-executable-aarch64 | |
| runs-on: | |
| - self-hosted | |
| - public | |
| - ARM64 | |
| - gprofiler | |
| - EC2 | |
| - linux | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Set up runner requirements | |
| run: scripts/setup_runner_requirements.sh | |
| # this gets GH_REPO and RELEASE_VERSION | |
| - name: Get and verify tag value | |
| run: ./scripts/verify_tag.sh | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Download executables from the previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_aarch64 | |
| path: output/ | |
| # TODO: Add docker layer caching when GitHub Actions cache is stabilized and works good with "satackey/[email protected]" | |
| - name: Build and push | |
| run: | | |
| set -x | |
| BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" | |
| AARCH64_IMAGE="$BASE_IMAGE-aarch64" | |
| # build & push Aarch64 | |
| # --skip-exe-build needs to be first! | |
| ./scripts/build_aarch64_container.sh --skip-exe-build --provenance=false --build-arg EXE_PATH=output/gprofiler_aarch64 -t "$AARCH64_IMAGE" --push | |
| deploy-container: | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-container-x64 | |
| - build-container-aarch64 | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: true | |
| # this gets GH_REPO and RELEASE_VERSION | |
| - name: Verify release tag matches gProfiler version | |
| run: ./scripts/verify_tag.sh | |
| # build-container-x64 has uploaded the image as an artifact, we donwload it here. | |
| # build-container-aarch64 has pushed the image to DockerHub, so we'll pull it later when creating | |
| # the manifest. | |
| - name: Download the x86_64 image from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gprofiler_x86_64.img | |
| path: output/ | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Push manifest | |
| run: | | |
| set -x | |
| BASE_IMAGE="${{ env.GH_REPO }}:${{ env.RELEASE_VERSION }}" | |
| LATEST_IMAGE="${{ env.GH_REPO }}:latest" | |
| AARCH64_IMAGE="$BASE_IMAGE-aarch64" | |
| X86_64_IMAGE="$BASE_IMAGE-x86_64" | |
| docker pull --platform=linux/aarch64 $AARCH64_IMAGE | |
| docker image load < output/gprofiler_x86_64.img | |
| docker tag gprofiler_x86_64 $X86_64_IMAGE | |
| docker push $X86_64_IMAGE | |
| # create manifests for the tag + for 'latest' | |
| docker buildx imagetools create -t "$BASE_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" | |
| docker buildx imagetools create -t "$LATEST_IMAGE" "$X86_64_IMAGE" "$AARCH64_IMAGE" |