chore: refactor to single multistage dockerfile with bake file (#754)

.github/workflows/docker-publish.yaml

@@ -47,12 +47,12 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # Build And Push Image
-      - name: Build Docker image
-        run: docker compose -f docker-compose-internal.yaml build
-      - name: Push Docker image
+      # Build And Push Images
+      - name: Build Docker images
+        run: docker buildx bake
+      - name: Push Docker images
         if: github.ref == 'refs/heads/master'
-        run: docker compose -f docker-compose-internal.yaml push
+        run: docker buildx bake --push
 
       - name: Send GitHub Action trigger data to Slack workflow
         if: ${{ failure() }}

Dockerfile

@@ -0,0 +1,109 @@
+# syntax=docker/dockerfile:1
+
+# Declare build arguments
+# TODO: this is only used for node image right now, should we also use it for nodeplugin?
+ARG SEMVER=""
+ARG GITCOMMIT=""
+ARG GITDATE=""
+
+FROM golang:1.21.1-alpine3.18 AS base-builder
+RUN apk add --no-cache make musl-dev linux-headers gcc git jq bash
+
+# Common build stage
+FROM base-builder AS common-builder
+WORKDIR /app
+COPY go.mod go.sum ./
+COPY disperser /app/disperser
+COPY common /app/common
+COPY core /app/core
+COPY api /app/api
+COPY contracts /app/contracts
+COPY indexer /app/indexer
+COPY encoding /app/encoding
+
+# Churner build stage
+FROM common-builder AS churner-builder
+COPY operators ./operators
+WORKDIR /app/operators
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/churner ./churner/cmd
+
+# Encoder build stage
+FROM common-builder AS encoder-builder
+WORKDIR /app/disperser
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/encoder ./cmd/encoder
+
+# API Server build stage
+FROM common-builder AS apiserver-builder
+WORKDIR /app/disperser
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/apiserver ./cmd/apiserver
+
+# Batcher build stage
+FROM common-builder AS batcher-builder
+WORKDIR /app/disperser
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/batcher ./cmd/batcher
+
+# Retriever build stage
+FROM common-builder AS retriever-builder
+COPY retriever /app/retriever
+COPY node /app/node
+COPY operators/churner /app/operators/churner
+WORKDIR /app/retriever
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/retriever ./cmd
+
+# Node build stage
+FROM common-builder AS node-builder
+COPY node /app/node
+COPY operators/churner /app/operators/churner
+WORKDIR /app/node
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -ldflags="-X 'github.com/Layr-Labs/eigenda/node.SemVer=${SEMVER}' -X 'github.com/Layr-Labs/eigenda/node.GitCommit=${GITCOMMIT}' -X 'github.com/Layr-Labs/eigenda/node.GitDate=${GITDATE}'" -o ./bin/node ./cmd
+
+# Nodeplugin build stage
+FROM common-builder AS node-plugin-builder
+COPY ./node /app/node
+COPY operators/churner /app/operators/churner
+WORKDIR /app/node
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go build -o ./bin/nodeplugin ./plugin/cmd
+
+
+# Final stages for each component
+FROM alpine:3.18 AS churner
+COPY --from=churner-builder /app/operators/bin/churner /usr/local/bin
+ENTRYPOINT ["churner"]
+
+FROM alpine:3.18 AS encoder
+COPY --from=encoder-builder /app/disperser/bin/encoder /usr/local/bin
+ENTRYPOINT ["encoder"]
+
+FROM alpine:3.18 AS apiserver
+COPY --from=apiserver-builder /app/disperser/bin/apiserver /usr/local/bin
+ENTRYPOINT ["apiserver"]
+
+FROM alpine:3.18 AS batcher
+COPY --from=batcher-builder /app/disperser/bin/batcher /usr/local/bin
+ENTRYPOINT ["batcher"]
+
+FROM alpine:3.18 AS retriever
+COPY --from=retriever-builder /app/retriever/bin/retriever /usr/local/bin
+ENTRYPOINT ["retriever"]
+
+FROM alpine:3.18 AS node
+COPY --from=node-builder /app/node/bin/node /usr/local/bin
+ENTRYPOINT ["node"]
+
+FROM alpine:3.18 AS nodeplugin
+COPY --from=node-plugin-builder /app/node/bin/nodeplugin /usr/local/bin
+ENTRYPOINT ["nodeplugin"]

Makefile

@@ -94,7 +94,8 @@ integration-tests-dataapi:
 	go test -v ./disperser/dataapi
 
 docker-release-build:
-	RELEASE_TAG=${SEMVER} docker compose -f docker-compose-release.yaml build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} ${PUSH_FLAG}
+	BUILD_TAG=${SEMVER} SEMVER=${SEMVER} GITCOMMIT=${GITCOMMIT} GITDATE=${GITDATE} \
+	docker buildx bake node-group-release ${PUSH_FLAG}
 
 semver:
 	echo "${SEMVER}"

docker-bake.hcl

@@ -0,0 +1,113 @@
+# VARIABLES
+
+variable "BUILD_TAG" {
+  default = "latest"
+}
+
+variable "SEMVER" {
+  default = "v0.0.0"
+}
+
+variable "GITCOMMIT" {
+  default = "dev"
+}
+
+variable "GITDATE" {
+  default = "0"
+}
+
+# GROUPS
+
+group "default" {
+  targets = ["all"]
+}
+
+group "all" {
+  targets = ["node-group", "disperser-group", "retriever", "churner"]
+}
+
+group "node-group" {
+  targets = ["node", "nodeplugin"]
+}
+
+group "disperser-group" {
+  targets = ["batcher", "disperser", "encoder"]
+}
+
+group "node-group-release" {
+  targets = ["node-release", "nodeplugin-release"]
+}
+
+# DISPERSER TARGETS
+
+target "batcher" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "batcher"
+  tags       = ["ghcr.io/layr-labs/eigenda/batcher:${BUILD_TAG}"]
+}
+
+target "disperser" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "apiserver"
+  tags       = ["ghcr.io/layr-labs/eigenda/disperser:${BUILD_TAG}"]
+}
+
+target "encoder" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "encoder"
+  tags       = ["ghcr.io/layr-labs/eigenda/encoder:${BUILD_TAG}"]
+}
+
+target "retriever" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "retriever"
+  tags       = ["ghcr.io/layr-labs/eigenda/retriever:${BUILD_TAG}"]
+}
+
+target "churner" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "churner"
+  tags       = ["ghcr.io/layr-labs/eigenda/churner:${BUILD_TAG}"]
+}
+
+# NODE TARGETS
+
+target "node" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "node"
+  tags       = ["ghcr.io/layr-labs/eigenda/node:${BUILD_TAG}"]
+  args = {
+    SEMVER    = "${SEMVER}"
+    GITCOMMIT = "${GITCOMMIT}"
+    GITDATE   = "${GITDATE}"
+  }
+}
+
+target "nodeplugin" {
+  context    = "."
+  dockerfile = "./Dockerfile"
+  target     = "nodeplugin"
+  tags       = ["ghcr.io/layr-labs/eigenda/nodeplugin:${BUILD_TAG}"]
+}
+
+# RELEASE TARGETS
+
+target "_release" {
+  platforms = ["linux/amd64", "linux/arm64"]
+}
+
+target "node-release" {
+  inherits = ["node", "_release"]
+  tags     = ["ghcr.io/layr-labs/eigenda/opr-node:${BUILD_TAG}"]
+}
+
+target "nodeplugin-release" {
+  inherits = ["nodeplugin", "_release"]
+  tags     = ["ghcr.io/layr-labs/eigenda/opr-nodeplugin:${BUILD_TAG}"]
+}