chore: cleanup docker targets in bake make ci (#1796)

* ci(test-proxy): fix docker build command

Previously it was testing the latest tag which is already pushed to ghcr, not the one built locally.

* chore(docker): cleanup bake/make/ci targets

Also add documentation for some of the random options/choices we made.

Author: samlaf

.github/workflows/docker-publish-release.yaml

@@ -68,9 +68,12 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
       if: ${{ success() }}
 
+    # We only push on `v*` tags or if the force input is true.
+    # We still run the build on every push to master just to ensure the images build correctly.
     - name: Set release PUSH_FLAG
       run: echo "PUSH_FLAG=--push" >> $GITHUB_ENV
       if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.force == 'true'
 
     - name: Build (and potentially push) docker image release
+      # The PUSH_FLAG is ingested by the Makefile and passed to docker buildx bake command.
       run: PUSH_FLAG=$PUSH_FLAG make docker-release-build

.github/workflows/docker-publish.yaml

@@ -50,10 +50,10 @@ jobs:
 
       # Build And Push Images
       - name: Build Docker images
-        run: docker buildx bake ci-release
+        run: docker buildx bake all
       - name: Push Docker images
         if: github.ref == 'refs/heads/master'
-        run: docker buildx bake ci-release --push
+        run: BUILD_TAG=master make docker-build-push
 
       - name: Send GitHub Action trigger data to Slack workflow
         if: ${{ failure() }}

.github/workflows/test-proxy.yml

@@ -189,7 +189,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: make docker-build
+      - run: BUILD_TAG=dev make docker-build
         working-directory: api/proxy
       # We also test that the docker container starts up correctly.
       - name: Run container as background process
@@ -200,7 +200,7 @@ jobs:
           -e EIGENDA_PROXY_ADDR=0.0.0.0 \
           -e EIGENDA_PROXY_PORT=6666 \
           -e EIGENDA_PROXY_MEMSTORE_ENABLED=true \
-          ghcr.io/layr-labs/eigenda-proxy:latest
+          ghcr.io/layr-labs/eigenda-proxy:dev
         working-directory: api/proxy
       - name: Wait for rpc to come up
         shell: bash

Makefile

@@ -121,9 +121,21 @@ integration-tests-dataapi:
 	make dataapi-build
 	go test -v ./disperser/dataapi
 
+# builds all services and loads them into dockerd (such that they are available via `docker images`).
+# The images will be tagged with :dev, which is the default BUILD_TAG in docker-bake.hcl.
+# This can be changed by running for example `BUILD_TAG=master make docker-build`.
+docker-build:
+	docker buildx bake all --load
+
+# builds all services and pushes them to the configured registry (ghcr by default).
+docker-build-push:
+	docker buildx bake all --push
+
+# Should only ever be used by the docker-publish-release CI workflow.
+# We keep the node-group and proxy targets separate since we might want to release them separately in the future.
 docker-release-build:
 	BUILD_TAG=${SEMVER} SEMVER=${SEMVER} GITDATE=${GITDATE} GIT_SHA=${GITSHA} GIT_SHORT_SHA=${GITCOMMIT} \
-	docker buildx bake node-group-release ${PUSH_FLAG} --provenance=false --sbom=false
+	docker buildx bake node-group-release ${PUSH_FLAG}
 	BUILD_TAG=${SEMVER} SEMVER=${SEMVER} GITDATE=${GITDATE} GIT_SHA=${GITSHA} GIT_SHORT_SHA=${GITCOMMIT} \
 	docker buildx bake proxy-release ${PUSH_FLAG}
 

api/proxy/Makefile

@@ -15,7 +15,7 @@ clean:
 
 docker-build:
 	# we only use this to build the docker image locally, so we give it the dev tag as a reminder
-	cd ../.. && BUILD_TAG=dev docker buildx bake proxy --load
+	cd ../.. && docker buildx bake proxy --load
 
 run-memstore-server: build
 	./bin/eigenda-proxy --memstore.enabled --metrics.enabled  --storage.backends-to-enable v2

docker-bake.hcl

@@ -7,8 +7,10 @@ variable "REPO" {
   default = "layr-labs/eigenda"
 }
 
+# We use the `dev` tag for local development builds.
+# CI builds will overwrite this with the `master` or `v*` tag.
 variable "BUILD_TAG" {
-  default = "latest"
+  default = "dev"
 }
 
 variable "SEMVER" {
@@ -47,37 +49,18 @@ group "all" {
     "traffic-generator-v2",
     "controller",
     "relay",
-    "blobapi"
+    "blobapi",
+    "proxy",
   ]
 }
 
 group "node-group" {
   targets = ["node", "nodeplugin"]
 }
 
-# Github public releases
-group "node-group-release" {
-  targets = ["node-release", "nodeplugin-release"]
-}
-
-# Github CI builds
-group "ci-release" {
-  targets = [
-    "node-group",
-    "batcher",
-    "disperser",
-    "encoder",
-    "retriever",
-    "churner",
-    "dataapi",
-    "controller",
-    "relay",
-    "blobapi",
-    "proxy",
-  ]
-}
-
-# Internal devops builds
+# Internal devops builds. These targets are used by the eigenda-devops CI pipeline.
+# TODO: refactor the ECR repo to make the `${REGISTRY}/${REPO}` tags such that we can
+# get rid of all of these internal targets.
 group "internal-release" {
   targets = [
     "node-internal",
@@ -294,7 +277,7 @@ target "proxy" {
   target     = "proxy"
   # We push to layr-labs/ directly instead of layr-labs/eigenda/ for historical reasons,
   # since proxy was previously in its own repo: https://github.com/Layr-Labs/eigenda-proxy
-  tags       = ["${REGISTRY}/layr-labs/proxy:${BUILD_TAG}"]
+  tags       = ["${REGISTRY}/layr-labs/eigenda-proxy:${BUILD_TAG}"]
 }
 
 target "proxy-internal" {
@@ -340,13 +323,19 @@ target "_release" {
   platforms = ["linux/amd64", "linux/arm64"]
 }
 
+group "node-group-release" {
+  targets = ["node-release", "nodeplugin-release"]
+}
+
 target "node-release" {
   inherits = ["node", "_release"]
+  # We overwrite the tag with a opr- prefix for public releases.
   tags     = ["${REGISTRY}/${REPO}/opr-node:${BUILD_TAG}"]
 }
 
 target "nodeplugin-release" {
   inherits = ["nodeplugin", "_release"]
+  # We overwrite the tag with a opr- prefix for public releases.
   tags     = ["${REGISTRY}/${REPO}/opr-nodeplugin:${BUILD_TAG}"]
 }