The API's coverage is measured, and every suite's is in Grafana (#394)
The cargo tests put web.rs at 17% and daemon.rs at 10%, because the browser suite, which drives the API, ran a daemon nobody measured. `sh scripts/coverage.sh api` builds an instrumented ipx and ipx-directory in the capped coverage container and runs the browser suite here against them (IPX_API_COVERAGE): web.rs is at 80% and daemon.rs at 77%. What it does not reach: access.rs (Cloudflare Access, 0%; its unit tests 73%) and the directory's public pages (site.rs, 4%). The coverage image is Ubuntu 24.04 now, as code-server is: built on Debian trixie, the binaries needed glibc 2.41 and code-server has 2.39. In that mode the suite starts those binaries rather than building its own, its tests' `ipx` commands too (global-setup's IPX_BIN), passes LLVM_PROFILE_FILE to both servers, and stops them with SIGTERM, since an instrumented binary writes its profile as it exits and Playwright otherwise kills them outright. Reports go to target/coverage/cargo, api and page. Each run ends with scripts/coverage-metrics.ts, which writes every suite's lines covered and in all, by crate, to node-exporter's textfile directory; the monitoring project's node-exporter now reads it (one flag, its compose file backed up beside it), so Prometheus keeps them as ipx_coverage_lines, and the iPX dashboard's Coverage row draws them: each suite now, over time, and by part. The cargo tests' totals hold their own code, which runs whole, so they read higher than the code alone; the panels say so. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
72
scripts/coverage-metrics.ts
Normal file
72
scripts/coverage-metrics.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
// The last coverage run of each suite, for Grafana (#394): lines the tests reach and lines there
|
||||
// are, by suite and part, as a file node-exporter's textfile collector serves to Prometheus.
|
||||
//
|
||||
// node scripts/coverage-metrics.ts (scripts/coverage.sh runs it after each report)
|
||||
//
|
||||
// Suites: `cargo` (the cargo tests, target/coverage/cargo), `browser` (the browser suite against an
|
||||
// instrumented ipx and ipx-directory, target/coverage/api) and `page` (web/src under the browser
|
||||
// suite, target/coverage/page). Parts: the crate, or `page`. Each run writes every suite that has
|
||||
// a report, so one suite's run leaves the others' numbers as they were.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const COVERAGE = path.join(import.meta.dirname, '..', 'target', 'coverage');
|
||||
// node-exporter reads it as /host/... (it mounts / at /host), on /mnt/fast beside its appdata.
|
||||
const OUT = process.env.IPX_COVERAGE_TEXTFILE ?? '/mnt/fast/appdata/node-exporter/textfile';
|
||||
|
||||
type Count = { covered: number, total: number };
|
||||
type Suite = { parts: Map<string, Count>, at: number };
|
||||
|
||||
/// The crate a path in cargo-llvm-cov's summary belongs to: directory/src/... is ipx-directory's.
|
||||
const crateOf = (file: string) => file.startsWith('directory/') ? 'ipx-directory' : file.startsWith('feed/') ? 'ipx-feed' : 'ipx';
|
||||
|
||||
function add(parts: Map<string, Count>, part: string, covered: number, total: number) {
|
||||
const c = parts.get(part) ?? { covered: 0, total: 0 };
|
||||
c.covered += covered; c.total += total;
|
||||
parts.set(part, c);
|
||||
}
|
||||
|
||||
/// cargo-llvm-cov's summary: a line a file, Lines and Missed Lines its 8th and 9th columns.
|
||||
function rust(dir: string): Suite | null {
|
||||
const file = path.join(COVERAGE, dir, 'summary.txt');
|
||||
if (!fs.existsSync(file)) return null;
|
||||
const parts = new Map<string, Count>();
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
const cols = line.trim().split(/\s+/);
|
||||
if (cols.length < 10 || !cols[0]!.endsWith('.rs')) continue;
|
||||
const lines = Number(cols[7]), missed = Number(cols[8]);
|
||||
if (Number.isFinite(lines) && Number.isFinite(missed)) add(parts, crateOf(cols[0]!), lines - missed, lines);
|
||||
}
|
||||
return { parts, at: fs.statSync(file).mtimeMs / 1000 };
|
||||
}
|
||||
|
||||
function page(): Suite | null {
|
||||
const file = path.join(COVERAGE, 'page', 'summary.json');
|
||||
if (!fs.existsSync(file)) return null;
|
||||
const { lines } = JSON.parse(fs.readFileSync(file, 'utf8')) as { lines: ({ file: string } & Count)[] };
|
||||
const parts = new Map<string, Count>();
|
||||
for (const l of lines) add(parts, 'page', l.covered, l.total);
|
||||
return { parts, at: fs.statSync(file).mtimeMs / 1000 };
|
||||
}
|
||||
|
||||
const suites: [string, Suite | null][] = [['cargo', rust('cargo')], ['browser', rust('api')], ['page', page()]];
|
||||
const out = [
|
||||
'# HELP ipx_coverage_lines Lines of iPX the tests reached and lines there are, as each suite\'s last coverage run found them (#394).',
|
||||
'# TYPE ipx_coverage_lines gauge',
|
||||
];
|
||||
for (const [suite, s] of suites) if (s) for (const [part, c] of s.parts) {
|
||||
out.push(`ipx_coverage_lines{suite="${suite}",part="${part}",state="covered"} ${c.covered}`);
|
||||
out.push(`ipx_coverage_lines{suite="${suite}",part="${part}",state="total"} ${c.total}`);
|
||||
}
|
||||
out.push('# HELP ipx_coverage_run_timestamp_seconds When each suite\'s last coverage report was made.');
|
||||
out.push('# TYPE ipx_coverage_run_timestamp_seconds gauge');
|
||||
for (const [suite, s] of suites) if (s) out.push(`ipx_coverage_run_timestamp_seconds{suite="${suite}"} ${Math.round(s.at)}`);
|
||||
|
||||
fs.mkdirSync(OUT, { recursive: true });
|
||||
// Written aside and renamed in, so a scrape never reads half a file.
|
||||
const file = path.join(OUT, 'ipx_coverage.prom');
|
||||
fs.writeFileSync(file + '.tmp', out.join('\n') + '\n');
|
||||
fs.renameSync(file + '.tmp', file);
|
||||
console.log(`coverage for Grafana: ${file}`);
|
||||
for (const [suite, s] of suites) if (s) for (const [part, c] of s.parts)
|
||||
console.log(` ${suite} ${part}: ${(100 * c.covered / c.total).toFixed(1)}% of ${c.total} lines`);
|
||||
@@ -1,17 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Which lines of the Rust the tests reach (#390), inside the limits builds are held to.
|
||||
# Which lines of the Rust the tests reach (#390, #394), inside the limits builds are held to.
|
||||
#
|
||||
# sh scripts/coverage.sh all three crates, then the report
|
||||
# sh scripts/coverage.sh ipx-feed one crate's tests, then the report of what has run
|
||||
# sh scripts/coverage.sh the cargo tests, all three crates, then their report
|
||||
# sh scripts/coverage.sh ipx-feed one crate's cargo tests, then the report of what has run
|
||||
# sh scripts/coverage.sh api the browser suite against an instrumented ipx and
|
||||
# ipx-directory: what the API and the daemon run (and the
|
||||
# page's coverage, which the same run measures)
|
||||
#
|
||||
# A coverage build instruments every crate, dependencies included, in a target of its own: a whole
|
||||
# rebuild, the kind that took Tower down on 2026-11-04. So it runs in a container under the same
|
||||
# rebuild, the kind that took Tower down on 2026-11-04. So it builds in a container under the same
|
||||
# cap as scripts/build-image.sh's builder (6 GB, six threads), past which the kernel kills it and
|
||||
# production is left alone; and, as CLAUDE.md asks of tests, one crate at a time. If the cap kills
|
||||
# it, say so and leave the cap be.
|
||||
# it, say so and leave the cap be. The browser suite runs here, as it always does, its servers the
|
||||
# container's build: the image is Ubuntu's, as code-server is, so they start here.
|
||||
#
|
||||
# The report: target/coverage/summary.txt (a line a file), target/coverage/lcov.info, and
|
||||
# target/coverage/html/index.html.
|
||||
# Reports: target/coverage/cargo and target/coverage/api (summary.txt, a line a file; lcov.info;
|
||||
# html/index.html), target/coverage/page (npm run coverage:page's, or api's). Each run ends by
|
||||
# writing the numbers for Grafana (scripts/coverage-metrics.ts).
|
||||
set -eu
|
||||
|
||||
MEM=6g
|
||||
@@ -21,13 +26,14 @@ MAX_LOAD=12
|
||||
IMAGE=192.168.1.130:5000/ipx-coverage:latest
|
||||
# The Docker daemon is Tower's, so its paths are Tower's: this tree is /mnt/user/vscode/ipx there,
|
||||
# and target/ is a link to target-real on /mnt/fast (builds through /mnt/user's FUSE took the load
|
||||
# to 248, CLAUDE.md).
|
||||
# to 248, CLAUDE.md). code-server sees /mnt/fast at the same path, so the instrumented binaries
|
||||
# and the profiles they write are at the same place for both.
|
||||
SRC=/mnt/user/vscode/ipx
|
||||
TARGET=/mnt/fast/vscode/ipx/target-real
|
||||
REGISTRY=$TARGET/coverage-registry
|
||||
LLVM_TARGET=$TARGET/llvm-cov-target
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
crates=${1:-ipx-feed ipx-directory ipx}
|
||||
|
||||
room() {
|
||||
avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)
|
||||
@@ -56,8 +62,45 @@ run() {
|
||||
}
|
||||
}
|
||||
|
||||
# The report of the profiles in the target, into target/coverage/$1, with the tree's own paths.
|
||||
report() {
|
||||
room
|
||||
mkdir -p "$TARGET/coverage/$1"
|
||||
run sh -c "cargo llvm-cov report --lcov --output-path /target/coverage/$1/lcov.info \
|
||||
&& cargo llvm-cov report --html --output-dir /target/coverage/$1 \
|
||||
&& cargo llvm-cov report --summary-only > /target/coverage/$1/summary.txt"
|
||||
# The paths inside the container are /work/...; the tree's own, from here.
|
||||
sed -i 's#/work/##' "$TARGET/coverage/$1/summary.txt" "$TARGET/coverage/$1/lcov.info"
|
||||
tail -1 "$TARGET/coverage/$1/summary.txt"
|
||||
}
|
||||
|
||||
docker pull -q "$IMAGE" >/dev/null
|
||||
mkdir -p "$REGISTRY"
|
||||
|
||||
if [ "${1:-}" = api ]; then
|
||||
# Built as cargo-llvm-cov builds for a test of its own, instrumented, with no tests: the
|
||||
# binaries the browser suite starts. Profiles from an earlier run are not this one's.
|
||||
run cargo llvm-cov clean --workspace
|
||||
for p in ipx-directory ipx; do
|
||||
room
|
||||
echo "== build $p"
|
||||
run sh -c "eval \"\$(cargo llvm-cov show-env --export-prefix)\" && cargo build --locked -p $p"
|
||||
done
|
||||
room
|
||||
echo "== the browser suite"
|
||||
# Not `set -e`'s to stop on: a failing test still ran what it ran, and the report says so; the
|
||||
# suite's own exit code is this script's at the end.
|
||||
suite=0
|
||||
IPX_API_COVERAGE=$LLVM_TARGET IPX_COVERAGE=1 LLVM_PROFILE_FILE="$LLVM_TARGET/ipx-browser-%p-%14m.profraw" \
|
||||
npx playwright test || suite=$?
|
||||
ls "$LLVM_TARGET"/ipx-browser-*.profraw >/dev/null 2>&1 \
|
||||
|| { echo "no profiles were written: did the servers stop on SIGTERM?" >&2; exit 1; }
|
||||
report api
|
||||
node scripts/coverage-metrics.ts
|
||||
exit "$suite"
|
||||
fi
|
||||
|
||||
crates=${1:-ipx-feed ipx-directory ipx}
|
||||
# A whole run starts from nothing; one crate adds to what has run.
|
||||
[ $# -eq 0 ] && run cargo llvm-cov clean --workspace
|
||||
for c in $crates; do
|
||||
@@ -65,11 +108,5 @@ for c in $crates; do
|
||||
echo "== $c"
|
||||
run cargo llvm-cov --no-report --locked -p "$c"
|
||||
done
|
||||
room
|
||||
mkdir -p "$TARGET/coverage"
|
||||
run sh -c 'cargo llvm-cov report --lcov --output-path /target/coverage/lcov.info \
|
||||
&& cargo llvm-cov report --html --output-dir /target/coverage \
|
||||
&& cargo llvm-cov report --summary-only > /target/coverage/summary.txt'
|
||||
# The paths inside the container are /work/...; the tree's own, from here.
|
||||
sed -i 's#/work/##' "$TARGET/coverage/summary.txt" "$TARGET/coverage/lcov.info"
|
||||
tail -1 "$TARGET/coverage/summary.txt"
|
||||
report cargo
|
||||
node scripts/coverage-metrics.ts
|
||||
|
||||
Reference in New Issue
Block a user