From 909573e1285404fb8802387ddd3f1a4a255cbdd1 Mon Sep 17 00:00:00 2001 From: Prasad Lohakpure Date: Fri, 10 Jul 2026 15:25:39 +0530 Subject: [PATCH] Enhancement: parallelize builds and native multi-arch Docker images Speed up CI/CD by parallelizing Go command/plugin builds and moving image builds off QEMU emulation. - build.sh: parallelize command + plugin builds via xargs with a bounded JOBS cap (avoids OOM on memory-limited runners); flag-driven steps (--go, --ui, --test) so each build stage runs independently. - Dockerfile: multi-stage build (native Go builder, web builder, slim runtime) with dependency-only cache layers for go mod + pnpm. - docker-image.yml: build each arch on its own native runner (no QEMU), push by digest, and stitch into one multi-arch tag with `docker buildx imagetools create`; GHA layer cache scoped per arch. - build.yml: native build + test on PRs with pnpm store caching. - README: document build.sh flags and env vars. --- .github/workflows/build.yml | 10 +++- .github/workflows/docker-image.yml | 88 +++++++++++++++++++++++++----- Dockerfile | 53 +++++++++++++++--- README.md | 15 +++++ build.sh | 44 +++++++++++---- 5 files changed, 175 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3311f80b..e2677250 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,5 +13,13 @@ jobs: uses: actions/setup-go@v5 with: go-version: 1.24.5 + # setup-go caches the module + build cache keyed on go.sum by default. + - name: Cache pnpm store + uses: actions/cache@v4 + with: + path: ~/.local/share/pnpm/store + key: ${{ runner.os }}-pnpm-${{ hashFiles('web/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm- - name: Build and Test - run: ./build.sh + run: ./build.sh --go --ui --test diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c1fa5c11..52e16d57 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,23 +8,80 @@ on: - "v*" - "latest" +env: + IMAGE: patternoss/heimdall + jobs: - docker-build: - runs-on: ubuntu-latest + # Build each arch on its OWN native runner (no QEMU) and push it "by digest" + # (an untagged, arch-specific image identified only by its sha256 digest). + build: + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up QEMU (for cross-platform builds) - uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to DockerHub + uses: docker/login-action@v3 with: - platforms: all + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_SUMMARY: false + DOCKER_BUILD_RECORD_UPLOAD: false + with: + context: . + platforms: ${{ matrix.platform }} + build-args: | + BUILD_VERSION=${{ steps.version.outputs.value }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Stitch: collect the per-arch digests and assemble a single tagged manifest list. + merge: + runs-on: ubuntu-latest + needs: build + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - with: - install: true - driver-opts: network=host - name: Log in to DockerHub uses: docker/login-action@v3 @@ -44,10 +101,11 @@ jobs: echo "DOCKER_TAG=latest" >> $GITHUB_ENV fi - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: patternoss/heimdall:${{ env.DOCKER_TAG }} - platforms: linux/amd64,linux/arm64 + - name: Create and push manifest list + working-directory: /tmp/digests + run: | + docker buildx imagetools create -t ${{ env.IMAGE }}:${{ steps.tagger.outputs.tag }} \ + $(printf '${{ env.IMAGE }}@sha256:%s ' *) + + - name: Inspect + run: docker buildx imagetools inspect ${{ env.IMAGE }}:${{ steps.tagger.outputs.tag }} diff --git a/Dockerfile b/Dockerfile index af76abbe..009c6201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,54 @@ -FROM golang:1.24.6 +FROM golang:1.24.6 AS go-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + binutils \ + binutils-gold \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y nodejs awscli jq -RUN apt-get update && apt-get install -y --no-install-recommends build-essential binutils binutils-gold pkg-config && rm -rf /var/lib/apt/lists/* WORKDIR /go/src/github.com/patterninc/heimdall -COPY . . +COPY go.mod go.sum ./ +RUN go mod download + +COPY build.sh ./ +COPY cmd ./cmd +COPY internal ./internal +COPY pkg ./pkg +COPY plugins ./plugins + +# Declared right before its only consumer so it invalidates as little as possible. +ARG BUILD_VERSION=dev +RUN BUILD_VERSION="${BUILD_VERSION}" ./build.sh --go + +FROM node:20-bookworm AS web-builder + +WORKDIR /app + +COPY web/pnpm-lock.yaml web/pnpm-workspace.yaml ./web/ +RUN corepack enable && corepack prepare pnpm@10.28.2 --activate && (cd web && pnpm fetch) + +COPY build.sh ./ +COPY web/ ./web/ +RUN ./build.sh --ui + +FROM node:20-bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + awscli \ + jq \ + && corepack enable && corepack prepare pnpm@10.28.2 --activate \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /go/src/github.com/patterninc/heimdall -# set config file +# Rarely-changing runtime files first, frequently-changing build outputs last. COPY configs/local.yaml /etc/heimdall/heimdall.yaml COPY entrypoint.sh /usr/local/bin/entrypoint.sh -# build executables -RUN ./build.sh +COPY assets ./assets +COPY --from=go-builder /go/src/github.com/patterninc/heimdall/dist ./dist +COPY --from=web-builder /app/web ./web CMD [ "/usr/local/bin/entrypoint.sh" ] \ No newline at end of file diff --git a/README.md b/README.md index 4ba0327f..553d320a 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,21 @@ GET /api/v1/job//stderr --- +## 🛠️ Building from Source + +`build.sh` compiles the Go binaries + plugins and builds the web UI. It builds **only what you ask for** via flags (no defaults): + +```bash +./build.sh --go # build Go binaries (cmd/*) + plugins (plugins/*.so) +./build.sh --ui # build the Next.js web UI +./build.sh --test # run the Go test suites +./build.sh --go --ui --test # do everything (what CI runs) +``` + +Passing no flag builds nothing. Unknown flags fail fast. Outputs land in `dist/` (binaries in `dist/`, plugins in `dist/plugins/`). + +--- + ## 🔌 Supported Plugins Heimdall supports a growing set of pluggable command types: diff --git a/build.sh b/build.sh index 790dc87c..64f9c8c6 100755 --- a/build.sh +++ b/build.sh @@ -7,25 +7,45 @@ set -ex BUILD_VERSION=$(git describe --tags --abbrev=0 --match "v*" | cut -dv -f2) WORKING_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) OUTPUT_DIR="${WORKING_DIR}/dist" -COMMANDS=$(ls ${WORKING_DIR}/cmd) -PLUGINS=$(ls ${WORKING_DIR}/plugins) LDFLAGS="-X main.Build=${BUILD_VERSION}" +BUILD_GO=false +BUILD_WEB=false +RUN_TESTS=false +for arg in "$@"; do + case "${arg}" in + --go) BUILD_GO=true ;; + --ui) BUILD_WEB=true ;; + --test) RUN_TESTS=true ;; + *) echo "unknown flag: ${arg} (use --go, --ui, --test)" >&2; exit 1 ;; + esac +done # reset output dir rm -rf ${OUTPUT_DIR} > /dev/null 2>&1 mkdir -p ${OUTPUT_DIR}/web ${OUTPUT_DIR}/plugins -# go environment -(cd ${WORKING_DIR} && go version) +if [ "${RUN_TESTS}" = "true" ]; then + echo "running tests" + (cd ${WORKING_DIR} && go test ./internal/pkg/... && go test ./pkg/...) +fi -# run tests -(cd ${WORKING_DIR} && go test ./internal/pkg/... && go test ./pkg/...) +if [ "${BUILD_GO}" = "true" ]; then + COMMANDS=$(ls ${WORKING_DIR}/cmd) + PLUGINS=$(ls ${WORKING_DIR}/plugins) -# build commands -(cd ${WORKING_DIR} && for item in ${COMMANDS}; do go build -ldflags "${LDFLAGS}" -o dist/$item cmd/$item/$item.go; done) -# build plugins -(cd ${WORKING_DIR} && for item in ${PLUGINS}; do go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/$item.so plugins/$item/$item.go; done) -# build web -(cd ${WORKING_DIR}/web && rm -rf node_modules > /dev/null 2>&1 && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) + # go environment + (cd ${WORKING_DIR} && go version) + + # build commands + (cd ${WORKING_DIR} && echo "${COMMANDS}" | xargs -P 4 -I {} bash -c 'echo "building command: {}" && go build -ldflags "${LDFLAGS}" -o dist/{} cmd/{}/{}.go') + + # build plugins + (cd ${WORKING_DIR} && echo "${PLUGINS}" | xargs -P 4 -I {} bash -c 'echo "building plugin: {}" && go build -buildmode=plugin -ldflags "${LDFLAGS}" -o dist/plugins/{}.so plugins/{}/{}.go') +fi + +if [ "${BUILD_WEB}" = "true" ]; then + # build web + (cd ${WORKING_DIR}/web && corepack enable && pnpm install --frozen-lockfile --ignore-scripts && pnpm run build) +fi