Images build in a capped BuildKit builder
A plain `docker buildx build` of ipx on 2026-11-04, its dependency layer invalidated by a Cargo.toml change (#382), rebuilt everything in release on all twelve threads with 5 GB free and no swap. Tower thrashed for three hours with every core pegged and had to be rebooted. scripts/build-image.sh now builds every image: it refuses under 9 GB available or above load 12, and builds in the docker-container builder ipx-capped, held to 6 GB and six threads by its cgroup, so a build that outgrows that is killed instead of stalling production. The builder pulls base images through mirror.gcr.io, which replaces tagging local copies to dodge Docker Hub's 429s. CLAUDE.md deploys only through it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
47
scripts/build-image.sh
Normal file
47
scripts/build-image.sh
Normal file
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
# Build and push one service's image, inside limits Tower can afford.
|
||||
#
|
||||
# sh scripts/build-image.sh ipx|directory|web
|
||||
#
|
||||
# On 2026-11-04 a plain `docker buildx build` of ipx (its dependencies' cache layer invalidated
|
||||
# by a Cargo.toml change) rebuilt everything in release on all twelve threads with 5 GB free and
|
||||
# no swap; Tower thrashed for three hours with every core pegged and had to be rebooted. So the
|
||||
# build runs in a BuildKit container under a cgroup cap: past the memory cap the build is killed,
|
||||
# it does not take production down with it. And it does not start without room for that cap.
|
||||
set -eu
|
||||
|
||||
MEM=6g # the builder's memory cap (Tower has no swap to spill into)
|
||||
CPUS=0-5 # six of twelve threads; cargo inside sees six and runs six jobs
|
||||
NEED_KB=$((9 * 1024 * 1024)) # the cap plus 3 GB for everything else
|
||||
MAX_LOAD=12
|
||||
BUILDER=ipx-capped
|
||||
|
||||
case "${1:-}" in
|
||||
ipx) FILE=Dockerfile; IMAGE=ipx ;;
|
||||
directory) FILE=Dockerfile.directory; IMAGE=ipx-directory ;;
|
||||
web) FILE=Dockerfile.web; IMAGE=ipx-web ;;
|
||||
*) echo "usage: $0 ipx|directory|web" >&2; exit 2 ;;
|
||||
esac
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
|
||||
load=$(cut -d' ' -f1 /proc/loadavg)
|
||||
echo "MemAvailable $((avail / 1024)) MB, load $load"
|
||||
if [ "$avail" -lt "$NEED_KB" ]; then
|
||||
echo "refusing: under $((NEED_KB / 1024 / 1024)) GB available. Wait, or find what is using it." >&2
|
||||
exit 1
|
||||
fi
|
||||
if awk -v l="$load" -v m="$MAX_LOAD" 'BEGIN { exit !(l > m) }'; then
|
||||
echo "refusing: load over $MAX_LOAD. Wait for it to come down." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Made once; buildx starts its container again when needed, after a reboot too.
|
||||
if ! docker buildx inspect "$BUILDER" >/dev/null 2>&1; then
|
||||
docker buildx create --name "$BUILDER" --driver docker-container \
|
||||
--driver-opt "memory=$MEM,cpuset-cpus=$CPUS" \
|
||||
--buildkitd-config scripts/buildkitd.toml >/dev/null
|
||||
fi
|
||||
|
||||
exec docker buildx build --builder "$BUILDER" -f "$FILE" \
|
||||
--tag "192.168.1.130:5000/$IMAGE:latest" --push .
|
||||
10
scripts/buildkitd.toml
Normal file
10
scripts/buildkitd.toml
Normal file
@@ -0,0 +1,10 @@
|
||||
# BuildKit's own config for the capped builder that scripts/build-image.sh makes.
|
||||
# The registry on Tower answers plain HTTP.
|
||||
[registry."192.168.1.130:5000"]
|
||||
http = true
|
||||
insecure = true
|
||||
|
||||
# This builder cannot see the images the Docker daemon has stored, so the base images come
|
||||
# from Google's mirror instead of Docker Hub, which rate-limits this host (429).
|
||||
[registry."docker.io"]
|
||||
mirrors = ["mirror.gcr.io"]
|
||||
Reference in New Issue
Block a user