cicd improvements

knights-analytics · Jul 18, 2024 · 4d16391 · 4d16391
1 parent db13ef6
commit 4d16391
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 14 deletions.
diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml
@@ -0,0 +1,49 @@
+name: Create image
+
+on: workflow_dispatch
+
+permissions:
+  contents: write
+  packages: write
+  checks: write
+  attestations: write
+  id-token: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+        - name: Build and push Docker image
+          id: push
+          uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+          with:
+            context: .
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+
+        * name: Generate artifact attestation
+          uses: actions/attest-build-provenance@v1
+          with:
+            subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+            subject-digest: ${{ steps.push.outputs.digest }}
+            push-to-registry: true
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -7,7 +7,14 @@ on:
 
 permissions:
   contents: write
+  packages: write
   checks: write
+  attestations: write
+  id-token: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   test:
@@ -56,6 +63,40 @@ jobs:
           report_paths: './testTarget/unit/*.xml'
           fail_on_failure: true
           require_tests: true
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      * name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
   release:
     name: Release
     runs-on: ubuntu-latest
@@ -78,11 +119,11 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: onnxruntime-linux-x64-gpu
-          path: .
+          path: ./onnxruntime-linux-x64-gpu
       - name: Display structure of downloaded files
         run: ls -R
       - uses: ncipollo/release-action@v1
         with:
-          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu.zip, hugot-cli-linux-x64"
+          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu, hugot-cli-linux-x64"
           generateReleaseNotes: true
           skipIfReleaseExists: true
diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ ARG GO_VERSION=1.22.5
 ARG RUST_VERSION=1.79
 ARG ONNXRUNTIME_VERSION=1.18.0
 ARG BUILD_PLATFORM=linux/amd64
-ARG CGO_LDFLAGS="-L./usr/lib/libtokenizers.a"
+
 #--- rust build of tokenizer ---
 
 FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer
@@ -55,7 +55,7 @@ RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w"
 # build cli binary
 COPY . /build
 WORKDIR /build
-RUN cd ./cmd && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
+RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
 
 # NON-PRIVILEDGED USER
 # create non-priviledged testuser with id: 1000

diff --git a/README.md b/README.md
@@ -45,7 +45,7 @@ Hugot can be used in two ways: as a library in your go application, or as a comm
 
 To use Hugot as a library in your application, you will need the following two dependencies on your system:
 
-- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it.
+- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it. Alternatively, you can explicitly specify the path to the folder with the `libtokenizers.a` file using the `CGO_LDFLAGS` env variable, see the [dockerfile](./Dockerfile).
 - the onnxruntime.go file obtained from the releases section of this page (if you want to use alternative architectures from `linux/amd64` you will have to download it from [the onnxruntime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example). Hugot looks for this file at /usr/lib/onnxruntime.so or /usr/lib64/onnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewSession()`, e.g:
 
 ```

diff --git a/scripts/run-unit-tests.sh b/scripts/run-unit-tests.sh
@@ -4,19 +4,23 @@ set -e
 
 # Directory of *this* script
 this_dir="$( cd "$( dirname "$0" )" && pwd )"
-export src_dir="$(realpath "${this_dir}/..")"
+src_dir="$(realpath "${this_dir}/..")"
+export src_dir
 
-export commit_hash=$(git rev-parse --short HEAD)
-export test_folder="$src_dir/testTarget"
-mkdir -p $test_folder
-export host_uid=$(id -u "$USER")
+commit_hash=$(git rev-parse --short HEAD)
+export commit_hash
+test_folder="$src_dir/testTarget"
+export test_folder
+mkdir -p "$test_folder"
+host_uid=$(id -u "$USER")
+export host_uid
 
 # build with compose
-docker compose -f $src_dir/compose-test.yaml build
+docker compose -f "$src_dir/compose-test.yaml" build
 echo "Running tests for commit hash: $commit_hash"
-docker compose -f $src_dir/compose-test.yaml up && \
-docker compose -f $src_dir/compose-test.yaml logs --no-color >& $test_folder/logs.txt
-docker compose -f $src_dir/compose-test.yaml rm -fsv
+docker compose -f "$src_dir/compose-test.yaml" up && \
+docker compose -f "$src_dir/compose-test.yaml" logs --no-color >& "$test_folder/logs.txt"
+docker compose -f "$src_dir/compose-test.yaml" rm -fsv
 
 echo "Extracting lib artifacts"
 docker build . --output "$src_dir/artifacts" --target artifacts