Skip to content

Commit

Permalink
Add infrastructure for generating the golang-debian image using upstr…
Browse files Browse the repository at this point in the history
…eam binaries (#1474)

* update dockerfile and Makefile to use upstream binaries for golang-debian image builds

* create a script for using builder base golang versions for tracking when pulling upstream binaries

* add script flag for releasing new upstream binary golang-debian images

* update prow_release_images to call new make targeting upstream binaries
  • Loading branch information
rcrozean authored Jul 30, 2024
1 parent 30ae0fc commit 88e3931
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 10 deletions.
36 changes: 36 additions & 0 deletions projects/golang/go/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ IMAGE_NAME?=golang-debian
IMAGE_TAG?=$(GIT_TAG)-$(BUILD_ID)-$(IMAGE_BUILD_ID)
LATEST_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(GIT_TAG)
IMAGE?=$(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG),$(LATEST_IMAGE)
# Variables used in the using upstream binarys in the golang-debian image
BUILDER_BASE_VERSIONS_YAML=$(BASE_DIRECTORY)/builder-base/versions.yaml
BUILDER_BASE_GO_VERSION=$(shell grep -E "^GOLANG_VERSION_$(subst .,,$(GO_SOURCE_VERSION))" $(BUILDER_BASE_VERSIONS_YAML))
GO_BIN_VERSION_WITH_RELEASE=$(subst GOLANG_VERSION_$(subst .,,$(GO_SOURCE_VERSION)): ,,$(BUILDER_BASE_GO_VERSION))
GO_BIN_VERSION_WITHOUT_RELEASE=$(shell [[ $(GO_BIN_VERSION_WITH_RELEASE) =~ [0-9]+.[0-9]+.[0-9]+ ]] && echo $${BASH_REMATCH[0]})

PUSH_IMAGES?=true
BUILDKIT_OUTPUT=type=image,oci-mediatypes=true,\"name=$(IMAGE)\",push=$(PUSH_IMAGES)
Expand Down Expand Up @@ -98,6 +103,10 @@ validate-go-archive-checksum:
fi ; \
done

.PHONY: prod-release-images-upstream-bins
prod-release-images-upstream-bins: export AWS_PROFILE=ecr-public-push
prod-release-images-upstream-bins: images-upstream-bins

.PHONY: fetch-golang-source-archive
fetch-golang-source-archive:
curl -L -o $(HOME)/rpmbuild/SOURCES/$(GIT_TAG).src.tar.gz https://github.com/golang/go/archive/refs/tags/$(GIT_TAG).tar.gz --create-dirs
Expand Down Expand Up @@ -144,6 +153,33 @@ images:
--progress plain \
--output $(BUILDKIT_OUTPUT)

.PHONY: fetch-golang-upstream-bins
fetch-golang-upstream-bins:
$(PROJECT_DIRECTORY)/scripts/get_upstream_golang.sh $(VERSION_DIRECTORY)/archives/ $(GO_BIN_VERSION_WITHOUT_RELEASE)

.PHONY: local-images-upstream-bins
local-images-upstream-bins: PUSH_IMAGES=false
local-images-upstream-bins: export BUILDKIT_HOST=docker-container://buildkitd
local-images-upstream-bins: images-upstream-bins

.PHONY: images-upstream-bins
images-upstream-bins: IMAGE_REPO=$(AWS_ACCOUNT_ID).dkr.ecr.$(AWS_REGION).amazonaws.com
images-upstream-bins: IMAGE_NAME=golang-debian
images-upstream-bins: IMAGE_TAG=$(GO_BIN_VERSION_WITHOUT_RELEASE)-$(BUILD_ID)-$(IMAGE_BUILD_ID)
images-upstream-bins: LATEST_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(GO_BIN_VERSION_WITHOUT_RELEASE)
images-upstream-bins: IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG),$(LATEST_IMAGE)
images-upstream-bins: fetch-golang-upstream-bins buildkit-check
images-upstream-bins:
$(BASE_DIRECTORY)/scripts/buildkit.sh \
build \
--frontend dockerfile.v0 \
--opt platform=$(GOOS)/$(ARCH_LOWER) \
--opt build-arg:GOLANG_ARCHIVE_PATH=$(GOOS)/$(ARCH_LOWER)/go$(GO_BIN_VERSION_WITHOUT_RELEASE).$(GOOS)-$(ARCH_LOWER).tar.gz \
--local dockerfile=$(PROJECT_DIRECTORY)/docker/debianBase \
--local context=$(VERSION_DIRECTORY)/archives \
--progress plain \
--output $(BUILDKIT_OUTPUT)

.PHONY: generate-golang-archive
generate-golang-archive: extract-golang-rpm tar-golang-sources

Expand Down
4 changes: 2 additions & 2 deletions projects/golang/go/docker/debianBase/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ COPY --from=go-untar /usr/local/go/ /usr/local/go/


ENV GOPATH /go
ENV PATH /usr/local/go/bin:$GOPATH/bin:$PATH
ENV PATH /usr/local/go/go/bin:$GOPATH/bin:$PATH

RUN set -eux; \
apt-get update; \
Expand All @@ -28,4 +28,4 @@ RUN set -eux; \
; \
go version

WORKDIR $GOPATH
WORKDIR $GOPATH
53 changes: 53 additions & 0 deletions projects/golang/go/scripts/get_upstream_golang.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -x
set -e
set -o pipefail

SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
BASE_DIRECTORY="$(git rev-parse --show-toplevel)"
GO_PREFIX="go"
ARCHITECTURE="linux/amd64" # Currently only build golang-debian image for amd64, if other arches are needed add

OUTPUT_DIR="$1"
GO_BIN_VERSION="$2"

source ${BASE_DIRECTORY}/builder-base/scripts/common_vars.sh

# Download from upstream and validate CHECKSUMs
function build::go::download {
# Set up specific go version by using go get, additional versions apart from default can be installed by calling
# the function again with the specific parameter.
local version=${1}
local outputDir=${2}
local archs=${3}

for arch in ${archs/,/ }; do
local filename="$outputDir/${arch}/go$version.${arch/\//-}.tar.gz"
if [ ! -f $filename ]; then
curl -sSLf --retry 5 "https://go.dev/dl/go$version.${arch/\//-}.tar.gz" -o $filename --create-dirs
sha256sum=$(curl -sSLf --retry 5 "https://go.dev/dl/?mode=json" | jq -r --arg tar "go$version.${arch/\//-}.tar.gz" '.[].files[] | if .filename == $tar then .sha256 else "" end' | xargs)

if [[ $(sha256sum ${filename} | cut -d ' ' -f1) != "${sha256sum}" ]]; then
echo "CHECKSUMs don't match"
exit 1
fi
fi
done
}

# strip the release version off the end of
build::go::download "${GO_BIN_VERSION}" "$OUTPUT_DIR" "$ARCHITECTURE"
16 changes: 8 additions & 8 deletions projects/golang/go/scripts/prow_release_images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,24 @@
# limitations under the License.

if [ "$ARCHITECTURE" == "ARM64" ]; then
echo "Won't perform image release for ARM64 arch"
exit 0
echo "Won't perform image release for ARM64 arch"
exit 0
fi

if [ "$AWS_ROLE_ARN" == "" ]; then
echo "Empty AWS_ROLE_ARN"
exit 1
echo "Empty AWS_ROLE_ARN"
exit 1
fi

if [ "$ECR_PUBLIC_PUSH_ROLE_ARN" == "" ]; then
echo "Empty ECR_PUBLIC_PUSH_ROLE_ARN"
exit 1
echo "Empty ECR_PUBLIC_PUSH_ROLE_ARN"
exit 1
fi

BASE_DIRECTORY=$(git rev-parse --show-toplevel)
cd ${BASE_DIRECTORY} || exit

cat << EOF > awscliconfig
cat <<EOF >awscliconfig
[default]
output=json
region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-west-2}}
Expand All @@ -47,4 +47,4 @@ export AWS_CONFIG_FILE=$(pwd)/awscliconfig
export AWS_PROFILE=ecr-public-push
unset AWS_ROLE_ARN AWS_WEB_IDENTITY_TOKEN_FILE

make -C ${BASE_DIRECTORY}/projects/golang/go prod-release-images
make -C ${BASE_DIRECTORY}/projects/golang/go "prod-release-images-upstream-bins"

0 comments on commit 88e3931

Please sign in to comment.