From c8cbb3047a3646d4861d41c6bca3a17e6686b83e Mon Sep 17 00:00:00 2001 From: rays Date: Wed, 4 Nov 2026 20:23:23 +0000 Subject: [PATCH] Images build in a capped BuildKit builder A plain `docker buildx build` of ipx on 2026-11-04, its dependency layer invalidated by a Cargo.toml change (#382), rebuilt everything in release on all twelve threads with 5 GB free and no swap. Tower thrashed for three hours with every core pegged and had to be rebooted. scripts/build-image.sh now builds every image: it refuses under 9 GB available or above load 12, and builds in the docker-container builder ipx-capped, held to 6 GB and six threads by its cgroup, so a build that outgrows that is killed instead of stalling production. The builder pulls base images through mirror.gcr.io, which replaces tagging local copies to dodge Docker Hub's 429s. CLAUDE.md deploys only through it. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 38 +++++++++++++++------------------- scripts/build-image.sh | 47 ++++++++++++++++++++++++++++++++++++++++++ scripts/buildkitd.toml | 10 +++++++++ 3 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 scripts/build-image.sh create mode 100644 scripts/buildkitd.toml diff --git a/CLAUDE.md b/CLAUDE.md index 0ee1b8d..81d8d1b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,9 +48,18 @@ Labels: `bug` for something wrong, `enhancement` for something missing. A proble for later stays open, and that is how it gets picked up again. Deploying a change is: build and push the image, then pull it and recreate the container. +**Build only with `scripts/build-image.sh`, never a bare `docker buildx build`.** On 2026-11-04 +a plain build of ipx, its dependency layer invalidated by a `Cargo.toml` change, rebuilt +everything in release on all twelve threads with 5 GB free; Tower thrashed for three hours with +every core pegged and had to be rebooted. The script refuses to start under 9 GB available or +above load 12, and builds in the BuildKit container `ipx-capped` (made on first use), held to 6 +GB of memory and six threads by its cgroup: a build that outgrows that is killed, and production +keeps running. If one is killed, say so and wait for the user to decide; do not raise the caps or go +around the script. Run the build in the foreground on its own, never chained after a cargo +command, and look at `MemAvailable` and the load again when it ends. ```sh -docker buildx build --tag 192.168.1.130:5000/ipx:latest . --push +sh scripts/build-image.sh ipx docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml pull ipx docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml up -d ipx docker logs --tail 20 iPX @@ -59,7 +68,7 @@ docker logs --tail 20 iPX The directory is the same, with its own image: ```sh -docker buildx build -f Dockerfile.directory --tag 192.168.1.130:5000/ipx-directory:latest . --push +sh scripts/build-image.sh directory docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml pull ipx-directory docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml up -d ipx-directory ``` @@ -67,7 +76,7 @@ docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml up -d ipx-directory And the page, `ipx-web`, the same again; a change to `web/` alone needs only this one: ```sh -docker buildx build -f Dockerfile.web --tag 192.168.1.130:5000/ipx-web:latest . --push +sh scripts/build-image.sh web docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml pull ipx-web docker compose -f /mnt/fast/arcane/projects/ipx/compose.yaml up -d ipx-web ``` @@ -94,24 +103,11 @@ block page, and ipx then logged a feed as failing with `invalid peer certificate **Name the service.** A bare `up -d` recreates every container in the project, which once the split lands means the frontend and directory too. Run `pull` before `up`, because `up` reuses whatever `latest` the host already has. -**A build that fails with `429 Too Many Requests` on a base image** is Docker Hub rate-limiting -this host. There is no Docker Hub login here, and the build asks about its base images -(`debian:trixie-slim` and `rust:1-slim-trixie`; `node:22-slim` and `caddy:2` for ipx-web) every time unless they are already stored locally. Pull them from Google's -mirror and tag them; the build then uses the local copies without asking Docker Hub: - -```sh -docker pull mirror.gcr.io/library/debian:trixie-slim -docker tag mirror.gcr.io/library/debian:trixie-slim debian:trixie-slim -docker pull mirror.gcr.io/library/rust:1-slim-trixie -docker tag mirror.gcr.io/library/rust:1-slim-trixie rust:1-slim-trixie -# ipx-web's, Dockerfile.web -docker pull mirror.gcr.io/library/node:22-slim -docker tag mirror.gcr.io/library/node:22-slim node:22-slim -docker pull mirror.gcr.io/library/caddy:2 -docker tag mirror.gcr.io/library/caddy:2 caddy:2 -``` - -Run those again now and then, or the local copies go stale. +**Base images come from Google's mirror** (`mirror.gcr.io`, set in `scripts/buildkitd.toml`), +not Docker Hub, which rate-limits this host with `429 Too Many Requests`. The capped builder +cannot see images the Docker daemon has stored, so pulling and tagging them locally does nothing +for it. If `buildkitd.toml` changes, remove the builder (`docker buildx rm ipx-capped`) so the +next build makes it again with the new config. The containers have no Docker healthcheck (#218): Docker runs one with `exec`, which on Tower fails whenever root is logged out, because elogind removes `/run/user/0`, where containerd puts diff --git a/scripts/build-image.sh b/scripts/build-image.sh new file mode 100644 index 0000000..0ef1e58 --- /dev/null +++ b/scripts/build-image.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Build and push one service's image, inside limits Tower can afford. +# +# sh scripts/build-image.sh ipx|directory|web +# +# On 2026-11-04 a plain `docker buildx build` of ipx (its dependencies' cache layer invalidated +# by a Cargo.toml change) rebuilt everything in release on all twelve threads with 5 GB free and +# no swap; Tower thrashed for three hours with every core pegged and had to be rebooted. So the +# build runs in a BuildKit container under a cgroup cap: past the memory cap the build is killed, +# it does not take production down with it. And it does not start without room for that cap. +set -eu + +MEM=6g # the builder's memory cap (Tower has no swap to spill into) +CPUS=0-5 # six of twelve threads; cargo inside sees six and runs six jobs +NEED_KB=$((9 * 1024 * 1024)) # the cap plus 3 GB for everything else +MAX_LOAD=12 +BUILDER=ipx-capped + +case "${1:-}" in + ipx) FILE=Dockerfile; IMAGE=ipx ;; + directory) FILE=Dockerfile.directory; IMAGE=ipx-directory ;; + web) FILE=Dockerfile.web; IMAGE=ipx-web ;; + *) echo "usage: $0 ipx|directory|web" >&2; exit 2 ;; +esac +cd "$(dirname "$0")/.." + +avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo) +load=$(cut -d' ' -f1 /proc/loadavg) +echo "MemAvailable $((avail / 1024)) MB, load $load" +if [ "$avail" -lt "$NEED_KB" ]; then + echo "refusing: under $((NEED_KB / 1024 / 1024)) GB available. Wait, or find what is using it." >&2 + exit 1 +fi +if awk -v l="$load" -v m="$MAX_LOAD" 'BEGIN { exit !(l > m) }'; then + echo "refusing: load over $MAX_LOAD. Wait for it to come down." >&2 + exit 1 +fi + +# Made once; buildx starts its container again when needed, after a reboot too. +if ! docker buildx inspect "$BUILDER" >/dev/null 2>&1; then + docker buildx create --name "$BUILDER" --driver docker-container \ + --driver-opt "memory=$MEM,cpuset-cpus=$CPUS" \ + --buildkitd-config scripts/buildkitd.toml >/dev/null +fi + +exec docker buildx build --builder "$BUILDER" -f "$FILE" \ + --tag "192.168.1.130:5000/$IMAGE:latest" --push . diff --git a/scripts/buildkitd.toml b/scripts/buildkitd.toml new file mode 100644 index 0000000..83fd0a8 --- /dev/null +++ b/scripts/buildkitd.toml @@ -0,0 +1,10 @@ +# BuildKit's own config for the capped builder that scripts/build-image.sh makes. +# The registry on Tower answers plain HTTP. +[registry."192.168.1.130:5000"] + http = true + insecure = true + +# This builder cannot see the images the Docker daemon has stored, so the base images come +# from Google's mirror instead of Docker Hub, which rate-limits this host (429). +[registry."docker.io"] + mirrors = ["mirror.gcr.io"]