Merge pull request #1845 from habitat-sh/sougata/saas_builder

Release SAAS builder built against LTS-2024 dependencies

.expeditor/scripts/verify/builder-api-functional.sh

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+# 10/11/2024: We need to use the most recent hab binary that is not yet in stable
+# to build against LTS without conflicts with existing stable packages
+hab pkg install core/hab --channel acceptance -bf
+
 echo "--- Generating signing key"
 hab origin key generate "$HAB_ORIGIN"
 
@@ -14,4 +18,6 @@ cp .secrets/habitat-env.sample .secrets/habitat-env
 echo "--- Entering studio"
 env HAB_NONINTERACTIVE=true \
     HAB_STUDIO_SUP=false \
+    HAB_INTERNAL_BLDR_CHANNEL=acceptance \
+    HAB_STUDIO_SECRET_HAB_INTERNAL_BLDR_CHANNEL=acceptance \
     hab studio enter

.expeditor/templates/studiorc

@@ -43,12 +43,6 @@ echo "--- Building changed builder components"
 echo "--- Building builder-api"
 echo "Redirecting log output; See build artifact 'builder-api.build.log'"
 build-builder api > logs/builder-api.build.log 2>&1
-echo "--- Building builder-jobsrv"
-echo "Redirecting log output; See build artifact 'builder-jobsrv.build.log'"
-build-builder jobsrv > logs/builder-jobsrv.build.log 2>&1
-echo "--- Building builder-worker"
-echo "Redirecting log output; See build artifact 'builder-worker.build.log'"
-build-builder worker > logs/builder-worker.build.log 2>&1
 
 echo "--- Waiting for services to start"
 while hab sup status | grep --quiet down;

.expeditor/verify.pipeline.yml

@@ -2,6 +2,10 @@ expeditor:
   defaults:
     buildkite:
       timeout_in_minutes: 30
+      env:
+          HAB_STUDIO_SECRET_HAB_BLDR_CHANNEL: LTS-2024
+          HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: LTS-2024
+          HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL: "LTS-2024"
 
 steps:
 #######################################################################
@@ -175,22 +179,7 @@ steps:
           privileged: true
           environment:
             - HAB_ORIGIN=habitat
-
-  - label: "[functional] :linux: builder-graph"
-    command:
-      - hab pkg install core/bats
-      - .expeditor/scripts/verify/builder-graph-functional.sh
-    artifact_paths:
-      - "logs/**/*"
-    retry:
-      automatic:
-        limit: 1
-    expeditor:
-      executor:
-        docker:
-          privileged: true
-          environment:
-            - HAB_ORIGIN=ci
+            - HAB_STUDIO_SECRET_NODE_OPTIONS="--dns-result-order=ipv4first"
 
   - label: "[build] :linux: builder-api-proxy"
     command:
@@ -209,5 +198,4 @@ steps:
             # `studio` should not matter.
             # test test
             - HAB_STUDIO_SECRET_NODE_OPTIONS="--dns-result-order=ipv4first"
-            - HAB_STUDIO_SECRET_HAB_BLDR_CHANNEL=LTS-2024
-            - HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL=LTS-2024
+

.github/actions/hab-pkg-build-and-upload-linux/action.yaml

@@ -15,6 +15,8 @@ runs:
       shell: bash
       env:
         BLDR_COMPONENT: ${{ inputs.bldr-component }}
+        HAB_BLDR_CHANNEL: LTS-2024
+        HAB_FALLBACK_CHANNEL: LTS-2024
       run: |
         hab pkg build $BLDR_COMPONENT
 

.studiorc

@@ -70,7 +70,8 @@ install-packages() {
   core/shadow \
   core/curl \
   core/aws-cli \
-  -b -c stable
+  core/sccache \
+  -b -c LTS-2024
 
   # workaround for https://github.com/habitat-sh/habitat/issues/6418
   hab pkg binlink core/cacerts

Makefile

@@ -1,7 +1,7 @@
 UNAME_S := $(shell uname -s)
 BIN = builder-graph
 LIB = builder-db builder-core github-api-client
-SRV = builder-api builder-jobsrv builder-worker
+SRV = builder-api
 ALL = $(BIN) $(LIB) $(SRV)
 
 .DEFAULT_GOAL := build-bin

build.sh

@@ -8,26 +8,30 @@ toolchain=$(get_toolchain)
 
 component=${1?component argument required}
 
+if [[ "$component" == "builder-jobsrv" || "$component" == "builder-worker" ]]; then
+  echo "Skipping build for $component as it is no longer supported."
+  exit 0
+fi
+
 # Accept hab license
-sudo hab license accept
-sudo hab pkg install core/rust/"$toolchain"
-sudo hab pkg install core/libarchive
-sudo hab pkg install core/openssl
-sudo hab pkg install core/zeromq
-sudo hab pkg install core/pkg-config
-sudo hab pkg install core/protobuf
-sudo hab pkg install core/postgresql
-sudo hab pkg install core/cmake
+sudo hab pkg install core/rust/"$toolchain" --channel LTS-2024
+sudo hab pkg install core/libarchive --channel LTS-2024
+sudo hab pkg install core/openssl --channel LTS-2024
+sudo hab pkg install core/zeromq --channel LTS-2024
+sudo hab pkg install core/pkg-config --channel LTS-2024
+sudo hab pkg install core/protobuf --channel LTS-2024
+sudo hab pkg install core/postgresql15 --channel LTS-2024
+sudo hab pkg install core/cmake --channel LTS-2024
 # It is important NOT to use a vendored openssl from openssl-sys
 # pg-sys does not use openssl-sys. So for components that use
 # diesel's postgres feature, you wil end up with 2 versions of openssl
 # which can lead to segmentation faults when connecting to postgres
 export OPENSSL_NO_VENDOR=1
 export LD_RUN_PATH
-LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib"
+LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql15)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib"
 export PKG_CONFIG_PATH
-PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig"
-eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql)/bin:$(hab pkg path core/cmake)/bin:$PATH"
+PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql15)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig"
+eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql15)/bin:$(hab pkg path core/cmake)/bin:$PATH"
 
 cd "components/$component"
 cargo build

components/builder-api/habitat/plan.sh

@@ -5,7 +5,7 @@ pkg_maintainer="The Habitat Maintainers <[email protected]>"
 pkg_license=('Apache-2.0')
 pkg_bin_dirs=(bin)
 pkg_deps=(core/glibc core/openssl core/coreutils core/gcc-libs core/zeromq
-core/libarchive core/curl core/postgresql)
+core/libarchive core/curl core/postgresql15)
 pkg_build_deps=(core/protobuf-cpp core/protobuf-rust core/coreutils core/cacerts
 core/rust/"$(tail -n 1 "../../../rust-toolchain"  | cut -d'"' -f 2)" core/gcc core/git core/pkg-config core/cmake)
 pkg_exports=(

components/builder-graph/habitat/plan.sh

@@ -10,7 +10,7 @@ pkg_deps=(
   core/openssl
   core/gcc-libs
   core/libarchive
-  core/postgresql
+  core/postgresql15
   core/zeromq #TODO: This can probably be removed if we removed the crate dep on builder-protocol
   core/zlib
   core/xz

support/ci/builder-dev-plan.sh

@@ -7,7 +7,6 @@ pkg_origin=habitat-dev
 do_dev_prepare() {
   # Order matters here
   export CARGO_HOME="/tmp/cargo_cache"
-  export builder_build_type="--debug"
   export RUSTC_WRAPPER
   RUSTC_WRAPPER="$(pkg_path_for core/sccache)/bin/sccache"
   export SCCACHE_DIR="/tmp/cargo_cache"