From d22d05555ca0d4714b46b6da306ef085807cb3e6 Mon Sep 17 00:00:00 2001 From: Thales Maciel Date: Fri, 17 Apr 2026 15:38:04 -0300 Subject: [PATCH] scripts: bundle-based golden image pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the OCI-push flow with a bundle-based one that mirrors the kernel catalog (publish-kernel.sh / kernelcat). - scripts/make-golden-bundle.sh: docker build → docker create → docker export | banger internal make-bundle → .tar.zst. Defaults target debian-bookworm / generic-6.12 / x86_64; pinned --size 4G to leave headroom for first-boot installs and in-VM apt use. - scripts/publish-golden-image.sh: rewritten to call make-golden-bundle, rclone upload to R2 (banger-images bucket, images.thaloco.com), and jq-patch internal/imagecat/catalog.json with URL / sha256 / size. --skip-upload stops after bundle build and copies to dist/. make-bundle default ext4 sizing also bumped from +25% to +50% headroom (mkfs.ext4 needs room for inode tables, block-group metadata, journal, and the default 5% reserved-blocks margin). The old 25% was too tight for the ~950 MB golden rootfs and aborted with "Could not allocate block". End-to-end smoke (local): golden Dockerfile → 286 MB tar.zst bundle with correct manifest, valid ext4, and all banger units + vsock agent present. Co-Authored-By: Claude Opus 4.7 (1M context) --- internal/cli/banger.go | 5 +- scripts/make-golden-bundle.sh | 120 +++++++++++++++++++ scripts/publish-golden-image.sh | 199 +++++++++++++++++++------------- 3 files changed, 243 insertions(+), 81 deletions(-) create mode 100755 scripts/make-golden-bundle.sh diff --git a/internal/cli/banger.go b/internal/cli/banger.go index 3ec07a7..6c129bf 100644 --- a/internal/cli/banger.go +++ b/internal/cli/banger.go @@ -426,7 +426,10 @@ func runInternalMakeBundle(cmd *cobra.Command, opts internalMakeBundleOpts) erro if err != nil { return fmt.Errorf("size rootfs tree: %w", err) } - sizeBytes = treeSize + treeSize/4 + // +50% headroom. mkfs.ext4 needs space for inode tables, + // block-group descriptors, journal, and the default 5% + // reserved-blocks margin on top of the raw data. + sizeBytes = treeSize + treeSize/2 if sizeBytes < imagepull.MinExt4Size { sizeBytes = imagepull.MinExt4Size } diff --git a/scripts/make-golden-bundle.sh b/scripts/make-golden-bundle.sh new file mode 100755 index 0000000..d45bb19 --- /dev/null +++ b/scripts/make-golden-bundle.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# make-golden-bundle.sh +# +# Build the banger golden image from images/golden/Dockerfile and +# package it as a .tar.zst bundle suitable for publishing to the +# imagecat catalog. Does not upload — see publish-golden-image.sh. +# +# Pipeline: +# docker build -> docker create -> docker export | banger internal make-bundle +# +# Usage: +# scripts/make-golden-bundle.sh [--name ] [--kernel-ref ] \ +# [--distro ] [--arch ] [--description "..."] \ +# [--out ] [--size ] [--platform

] +# +# Defaults: +# --name debian-bookworm +# --kernel-ref generic-6.12 +# --distro debian +# --arch x86_64 +# --platform linux/amd64 +# --out /dist/-.tar.zst +# +# Environment overrides: +# BANGER_BIN path to banger binary (default build/bin/banger) +# BANGER_VSOCK_AGENT_BIN path to companion (default build/bin/banger-vsock-agent) + +set -euo pipefail + +log() { printf '[make-golden-bundle] %s\n' "$*" >&2; } +die() { log "$*"; exit 1; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DOCKERFILE="$REPO_ROOT/images/golden/Dockerfile" +CONTEXT="$REPO_ROOT/images/golden" + +NAME="debian-bookworm" +KERNEL_REF="generic-6.12" +DISTRO="debian" +ARCH="x86_64" +DESCRIPTION="" +OUT="" +# 4G is a deliberate over-allocation for the golden image: it leaves +# room for first-boot apt-installs of sshd on derived pulls and for +# the user's own apt-installs during sandbox use. +SIZE="4G" +PLATFORM="linux/amd64" + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) NAME="${2:-}"; shift 2;; + --kernel-ref) KERNEL_REF="${2:-}"; shift 2;; + --distro) DISTRO="${2:-}"; shift 2;; + --arch) ARCH="${2:-}"; shift 2;; + -d|--description) DESCRIPTION="${2:-}"; shift 2;; + --out) OUT="${2:-}"; shift 2;; + --size) SIZE="${2:-}"; shift 2;; + --platform) PLATFORM="${2:-}"; shift 2;; + -h|--help) + sed -n '2,/^set -euo/p' "$0" | sed 's/^# \?//' | sed '$d' + exit 0 + ;; + *) die "unknown option: $1";; + esac +done + +for tool in docker zstd sha256sum; do + command -v "$tool" >/dev/null 2>&1 || die "missing required tool: $tool" +done +[[ -f "$DOCKERFILE" ]] || die "dockerfile missing: $DOCKERFILE" + +BANGER_BIN="${BANGER_BIN:-$REPO_ROOT/build/bin/banger}" +[[ -x "$BANGER_BIN" ]] || die "banger binary not executable: $BANGER_BIN (run 'make build' or set BANGER_BIN)" +VSOCK_AGENT="${BANGER_VSOCK_AGENT_BIN:-$REPO_ROOT/build/bin/banger-vsock-agent}" +[[ -x "$VSOCK_AGENT" ]] || die "banger-vsock-agent not executable: $VSOCK_AGENT (run 'make build')" + +if [[ -z "$OUT" ]]; then + OUT="$REPO_ROOT/dist/${NAME}-${ARCH}.tar.zst" +fi +mkdir -p "$(dirname "$OUT")" + +DOCKER_TAG="banger-golden:${NAME}" + +log "building $DOCKER_TAG (platform=$PLATFORM)" +docker build --platform "$PLATFORM" -t "$DOCKER_TAG" -f "$DOCKERFILE" "$CONTEXT" + +log "creating docker container (not started)" +CONTAINER_ID="$(docker create "$DOCKER_TAG")" +cleanup() { docker rm -f "$CONTAINER_ID" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +log "piping container filesystem into banger internal make-bundle" +SIZE_FLAG=() +[[ -n "$SIZE" ]] && SIZE_FLAG=(--size "$SIZE") +DESC_FLAG=() +[[ -n "$DESCRIPTION" ]] && DESC_FLAG=(--description "$DESCRIPTION") +KERNEL_REF_FLAG=() +[[ -n "$KERNEL_REF" ]] && KERNEL_REF_FLAG=(--kernel-ref "$KERNEL_REF") + +export BANGER_VSOCK_AGENT_BIN="$VSOCK_AGENT" +docker export "$CONTAINER_ID" | \ + "$BANGER_BIN" internal make-bundle \ + --rootfs-tar - \ + --name "$NAME" \ + --distro "$DISTRO" \ + --arch "$ARCH" \ + "${KERNEL_REF_FLAG[@]}" \ + "${DESC_FLAG[@]}" \ + "${SIZE_FLAG[@]}" \ + --out "$OUT" + +SHA256="$(sha256sum "$OUT" | awk '{print $1}')" +SIZE_BYTES="$(stat -c '%s' "$OUT")" +HUMAN="$(numfmt --to=iec --suffix=B "$SIZE_BYTES" 2>/dev/null || echo "${SIZE_BYTES}B")" + +log "bundle: $OUT" +log "sha256: $SHA256" +log "size: $HUMAN ($SIZE_BYTES bytes)" +printf '%s\n' "$OUT" diff --git a/scripts/publish-golden-image.sh b/scripts/publish-golden-image.sh index 2b7606b..8ca65b2 100755 --- a/scripts/publish-golden-image.sh +++ b/scripts/publish-golden-image.sh @@ -1,104 +1,143 @@ #!/usr/bin/env bash -# Build and optionally push the banger golden image. +# publish-golden-image.sh # -# Examples: -# ./scripts/publish-golden-image.sh --tag thaloco/banger-golden:debian-bookworm -# ./scripts/publish-golden-image.sh --tag thaloco/banger-golden:debian-bookworm --push -# ./scripts/publish-golden-image.sh --tag ghcr.io/thaloco/banger-golden:latest --push --platform linux/amd64 +# Build the banger golden-image bundle, upload it to R2, and patch +# internal/imagecat/catalog.json with the resulting URL + sha256 + +# size. Mirrors publish-kernel.sh for kernelcat. # -# The script expects the user to be logged in to the target registry -# (docker login / gh auth token) when --push is set. +# Usage: +# scripts/publish-golden-image.sh [--name ] [--kernel-ref ] \ +# [--distro ] [--arch ] [--description "..."] \ +# [--size ] [--platform

] [--skip-upload] +# +# Environment overrides: +# RCLONE_REMOTE rclone remote to upload through (default: r2) +# RCLONE_BUCKET R2 bucket name (default: banger-images) +# BASE_URL public URL prefix for the bucket (default: https://images.thaloco.com) set -euo pipefail +log() { printf '[publish-golden-image] %s\n' "$*" >&2; } +die() { log "$*"; exit 1; } + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -DOCKERFILE="$REPO_ROOT/images/golden/Dockerfile" -CONTEXT="$REPO_ROOT/images/golden" +CATALOG_FILE="$REPO_ROOT/internal/imagecat/catalog.json" -TAG="" -PUSH=0 +RCLONE_REMOTE="${RCLONE_REMOTE:-r2}" +RCLONE_BUCKET="${RCLONE_BUCKET:-banger-images}" +BASE_URL="${BASE_URL:-https://images.thaloco.com}" + +NAME="debian-bookworm" +KERNEL_REF="generic-6.12" +DISTRO="debian" +ARCH="x86_64" +DESCRIPTION="" +SIZE="" PLATFORM="linux/amd64" -EXTRA_TAGS=() - -usage() { - cat <<'EOF' -Usage: publish-golden-image.sh --tag [--tag ] [--push] [--platform ] - -Options: - --tag Primary image reference (required). Repeat --tag for extra tags - (e.g. to publish both :latest and :debian-bookworm). - --push Push all tags after building. Requires prior `docker login`. - --platform Build platform (default: linux/amd64). banger x86_64-only today. - -h, --help This help. -EOF -} +SKIP_UPLOAD=0 while [[ $# -gt 0 ]]; do case "$1" in - --tag) - if [[ -z "$TAG" ]]; then - TAG="${2:-}" - else - EXTRA_TAGS+=("${2:-}") - fi - shift 2 - ;; - --push) - PUSH=1 - shift - ;; - --platform) - PLATFORM="${2:-}" - shift 2 - ;; + --name) NAME="${2:-}"; shift 2;; + --kernel-ref) KERNEL_REF="${2:-}"; shift 2;; + --distro) DISTRO="${2:-}"; shift 2;; + --arch) ARCH="${2:-}"; shift 2;; + -d|--description) DESCRIPTION="${2:-}"; shift 2;; + --size) SIZE="${2:-}"; shift 2;; + --platform) PLATFORM="${2:-}"; shift 2;; + --skip-upload) SKIP_UPLOAD=1; shift;; -h|--help) - usage + sed -n '2,/^set -euo/p' "$0" | sed 's/^# \?//' | sed '$d' exit 0 ;; - *) - echo "unknown option: $1" >&2 - usage >&2 - exit 1 - ;; + *) die "unknown option: $1";; esac done -if [[ -z "$TAG" ]]; then - echo "--tag is required" >&2 - usage >&2 - exit 1 -fi - -if ! command -v docker >/dev/null 2>&1; then - echo "docker binary not found in PATH" >&2 - exit 1 -fi - -BUILD_ARGS=(build --platform "$PLATFORM" -t "$TAG" -f "$DOCKERFILE") -for t in "${EXTRA_TAGS[@]}"; do - BUILD_ARGS+=(-t "$t") +for tool in jq sha256sum stat; do + command -v "$tool" >/dev/null 2>&1 || die "missing required tool: $tool" done -BUILD_ARGS+=("$CONTEXT") - -echo "==> building $TAG (platform=$PLATFORM)" -docker "${BUILD_ARGS[@]}" - -if [[ "$PUSH" -eq 1 ]]; then - echo "==> pushing $TAG" - docker push "$TAG" - for t in "${EXTRA_TAGS[@]}"; do - echo "==> pushing $t" - docker push "$t" +[[ -f "$CATALOG_FILE" ]] || die "catalog file not found: $CATALOG_FILE" +if [[ "$SKIP_UPLOAD" -ne 1 ]]; then + for tool in rclone curl; do + command -v "$tool" >/dev/null 2>&1 || die "missing required tool: $tool" done fi -echo "==> done" -echo " primary tag: $TAG" -for t in "${EXTRA_TAGS[@]}"; do - echo " extra tag : $t" -done -if [[ "$PUSH" -eq 0 ]]; then - echo - echo "Image is built locally but not pushed. Pass --push to publish." +TARBALL_NAME="${NAME}-${ARCH}.tar.zst" +STAGE="$(mktemp -d)" +trap 'rm -rf "$STAGE"' EXIT +OUT="$STAGE/$TARBALL_NAME" + +log "building bundle via make-golden-bundle.sh" +SIZE_FLAG=() +[[ -n "$SIZE" ]] && SIZE_FLAG=(--size "$SIZE") +"$SCRIPT_DIR/make-golden-bundle.sh" \ + --name "$NAME" \ + --kernel-ref "$KERNEL_REF" \ + --distro "$DISTRO" \ + --arch "$ARCH" \ + --description "$DESCRIPTION" \ + --platform "$PLATFORM" \ + "${SIZE_FLAG[@]}" \ + --out "$OUT" + +SHA256="$(sha256sum "$OUT" | awk '{print $1}')" +SIZE_BYTES="$(stat -c '%s' "$OUT")" +HUMAN="$(numfmt --to=iec --suffix=B "$SIZE_BYTES" 2>/dev/null || echo "${SIZE_BYTES}B")" +log "bundle ready: $TARBALL_NAME ($HUMAN, sha256 $SHA256)" + +if [[ "$SKIP_UPLOAD" -eq 1 ]]; then + KEEP="$REPO_ROOT/dist/$TARBALL_NAME" + mkdir -p "$(dirname "$KEEP")" + cp -f "$OUT" "$KEEP" + log "--skip-upload set; catalog not patched" + log "bundle kept at: $KEEP" + exit 0 fi + +log "uploading to $RCLONE_REMOTE:$RCLONE_BUCKET/$TARBALL_NAME" +rclone copyto "$OUT" "$RCLONE_REMOTE:$RCLONE_BUCKET/$TARBALL_NAME" + +URL="$BASE_URL/$TARBALL_NAME" +log "verifying $URL is reachable" +HEAD_STATUS="$(curl -fsSI -o /dev/null -w '%{http_code}' "$URL" || true)" +if [[ "$HEAD_STATUS" != "200" ]]; then + die "uploaded tarball is not publicly reachable at $URL (HTTP $HEAD_STATUS); check bucket public-access config" +fi + +log "patching $CATALOG_FILE" +NEW_ENTRY="$(jq -n \ + --arg name "$NAME" \ + --arg distro "$DISTRO" \ + --arg arch "$ARCH" \ + --arg kref "$KERNEL_REF" \ + --arg url "$URL" \ + --arg sha "$SHA256" \ + --argjson size "$SIZE_BYTES" \ + --arg desc "$DESCRIPTION" \ + '{ + name: $name, + distro: $distro, + arch: $arch, + kernel_ref: $kref, + tarball_url: $url, + tarball_sha256: $sha, + size_bytes: $size, + description: $desc + } | with_entries(select(.value != null and .value != ""))')" + +CATALOG_TMP="$(mktemp)" +jq --arg name "$NAME" --argjson new "$NEW_ENTRY" ' + .version = (.version // 1) + | .entries = (((.entries // []) | map(select(.name != $name))) + [$new]) + | .entries |= sort_by(.name) +' "$CATALOG_FILE" > "$CATALOG_TMP" +mv "$CATALOG_TMP" "$CATALOG_FILE" + +log "done" +log "next steps:" +log " git diff -- $CATALOG_FILE" +log " git add $CATALOG_FILE && git commit -m 'imagecat: publish $NAME'" +log " make build # rebuild banger so the new catalog is embedded"