diff --git a/CLAUDE.md b/CLAUDE.md index b9190d8..113ea6b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -209,6 +209,7 @@ node tests/contrast.ts # every theme's palette against WCAG AA npx playwright test # the browser tests, against a real daemon on fixture feeds node tests/load/run.ts # k6: many people at once against a scratch daemon with 1,500 feeds sh scripts/coverage.sh # which Rust lines the cargo tests reach, capped like a build (#390) +sh scripts/coverage.sh api # which Rust lines the browser suite reaches through the API (#394) npm run coverage:page # which lines of web/src the browser tests reach (#390) ``` @@ -287,17 +288,26 @@ seeking through files, a flood of wrong passwords. Things about them: * The Rust tests run on Postgres with the same URL: `. /src/.envrc && cargo test`. Each test makes a schema `ipxt__`, and the next run drops those an earlier one left. -Coverage (#390) says what the tests leave untested; read it before deciding a change is covered. -`scripts/coverage.sh` runs the Rust tests instrumented in the `ipx-coverage` image +Coverage (#390, #394) says what the tests leave untested; read it before deciding a change is +covered. `sh scripts/coverage.sh` runs the cargo tests instrumented in the `ipx-coverage` image (`sh scripts/build-image.sh coverage` makes it), in a container held to the builder's 6 GB and six -threads, one crate at a time, and writes `target/coverage/summary.txt`, `lcov.info` and -`html/`. Its first run rebuilds every dependency instrumented, in a target of its own; later -ones only what changed. It measures the cargo tests alone: the API handlers (`web.rs`) and the daemon run -under the browser suite, whose daemon is not instrumented, so their low numbers there say little. -`npm run coverage:page` is the browser suite on a build of the page that keeps a source map to -each `web/src` file (`IPX_COVERAGE=1`, `web/concat-map.mts`), and writes `target/coverage/page`. -The specs take `test` and `expect` from `tests/ui/coverage.ts`, not from `@playwright/test`, so -every test is counted. The service worker, the unit tests and the native-bridge check are not. +threads, one crate at a time, into `target/coverage/cargo` (`summary.txt`, `lcov.info`, `html/`). +`sh scripts/coverage.sh api` builds an instrumented ipx and ipx-directory there and runs the +browser suite here against them (`IPX_API_COVERAGE`), into `target/coverage/api`: what the API +and the daemon really run, which the cargo tests alone put at 17% for `web.rs`. The image is +Ubuntu 24.04, as code-server is, or the binaries it builds would not start here (glibc); the +suite stops its servers with SIGTERM in that mode, since an instrumented binary writes its profile +as it exits. The same run measures the page; `npm run coverage:page` does that alone, on a build of +the page with a source map to each `web/src` file (`IPX_COVERAGE=1`, `web/concat-map.mts`), into +`target/coverage/page`. The specs take `test` and `expect` from `tests/ui/coverage.ts`, not from +`@playwright/test`, so every test is counted. The service worker, the unit tests and the +native-bridge check are not. + +Each run ends with `node scripts/coverage-metrics.ts`, which writes every suite's last numbers to +`/mnt/fast/appdata/node-exporter/textfile/ipx_coverage.prom`; node-exporter's textfile collector +(the monitoring project's compose file) serves them as `ipx_coverage_lines{suite,part,state}`, job +`node`, and the iPX dashboard's Coverage row draws them. A coverage run started by hand updates +Grafana; nothing runs one on its own. Non-trivial logic leaves one runnable check behind. Pure functions (`merge_policy`, `pick`, `matches_keywords`, `parse_interval`) are the easiest place to put it. diff --git a/Dockerfile.coverage b/Dockerfile.coverage index c4c1fb4..9409978 100644 --- a/Dockerfile.coverage +++ b/Dockerfile.coverage @@ -2,17 +2,22 @@ # toolchain, LLVM's coverage tools and cargo-llvm-cov. No source: coverage.sh mounts the tree. # Built and pushed like the services, by scripts/build-image.sh coverage, under its caps. # +# Ubuntu 24.04, as code-server is (#394): `coverage.sh api` builds an instrumented ipx and +# ipx-directory here, under the cap, and the browser suite runs them in code-server, which a +# binary linked against Debian trixie's newer glibc (2.41 to Ubuntu's 2.39) would not start in. # The same Rust as the tests are built with elsewhere: a newer one would rebuild everything with # its own flags for nothing. -FROM rust:1.98.1-slim-trixie +FROM ubuntu:24.04 +ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo PATH=/usr/local/cargo/bin:$PATH # cc for rusqlite's bundled SQLite and librqbit, as Dockerfile's builder; ffmpeg for DeadAir's -# tests, as the image ipx runs in. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential ffmpeg \ +# tests, as the image ipx runs in; curl and certificates to fetch rustup. +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl ffmpeg \ && rm -rf /var/lib/apt/lists/* \ + && curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.1 --no-modify-path \ && rustup component add llvm-tools-preview # cargo-llvm-cov's own build, as its README gives it, rather than compiling it here. ADD https://github.com/taiki-e/cargo-llvm-cov/releases/latest/download/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz /tmp/llvm-cov.tar.gz RUN tar -xzf /tmp/llvm-cov.tar.gz -C /usr/local/cargo/bin && rm /tmp/llvm-cov.tar.gz \ - && cargo llvm-cov --version + && cargo llvm-cov --version && rustc --version WORKDIR /work diff --git a/grafana/dashboard.py b/grafana/dashboard.py index 3ae8df3..fb4ee87 100644 --- a/grafana/dashboard.py +++ b/grafana/dashboard.py @@ -417,6 +417,37 @@ panel("logs", "ipx log", 24, 12, 0, description="Without the event stream's requests, not-due skips and healthcheck status calls.", options=LOG_OPTS) y += 12 +# ---- Coverage: what the tests reach (#390, #394). scripts/coverage.sh writes each run's numbers +# for node-exporter's textfile collector, so they are job node's, and stay as the last run left them. +COV = 'ipx_coverage_lines{job="node"' +def cov(suite, by=""): + """Lines covered over lines there are, for a suite, summed over its parts or by `by`.""" + group = f" by ({by})" if by else "" + return (f'sum{group} ({COV},suite="{suite}",state="covered"}}) / ' + f'sum{group} ({COV},suite="{suite}",state="total"}})') +row("Coverage") +COV_STEPS = [{"color": "red", "value": None}, {"color": "orange", "value": 0.6}, {"color": "green", "value": 0.8}] +stat("Rust, cargo tests", prom(cov("cargo")), 0, unit="percentunit", thresholds=COV_STEPS, + desc="Lines of ipx, ipx-directory and ipx-feed the cargo tests reach: sh scripts/coverage.sh. The tests' own code counts too, as it runs whole, so this reads higher than the code alone would.") +stat("Rust, browser tests", prom(cov("browser")), 4, unit="percentunit", thresholds=COV_STEPS, + desc="Lines of ipx and ipx-directory the browser suite reaches through the API, against an instrumented build: sh scripts/coverage.sh api.") +stat("Page, browser tests", prom(cov("page")), 8, unit="percentunit", thresholds=COV_STEPS, + desc="Lines of web/src the browser suite runs: npm run coverage:page, or coverage.sh api.") +stat("Since the last coverage run", prom('time() - max(ipx_coverage_run_timestamp_seconds{job="node"})'), 12, unit="s", + desc="The newest of the three suites' reports. Each suite's own is in the table beside it.") +table("Last run of each suite", [prom('time() - ipx_coverage_run_timestamp_seconds{job="node"}', instant=True)], 16, w=8, h=4, + rename={"Value": "ago", "suite": "suite"}, unit="s") +y += 4 +ts("Lines covered", [prom(cov("cargo"), legend="Rust, cargo tests"), prom(cov("browser"), ref="B", legend="Rust, browser tests"), + prom(cov("page"), ref="C", legend="Page, browser tests")], + 0, w=12, unit="percentunit", desc="As each suite's last run found them: a step is a run that found more, or less.") +table("By part", [prom(cov("cargo", "part"), ref="A", instant=True), prom(cov("browser", "part"), ref="B", instant=True), + prom(cov("page", "part"), ref="C", instant=True)], + 12, w=12, unit="percentunit", rename={"Value #A": "cargo tests", "Value #B": "browser tests", "Value #C": "page"}, + desc="Each crate's lines, and the page's, by the suite that reached them. A crate has more lines under the cargo tests: their own code, which a build for the browser suite leaves out.") +panels[-1]["transformations"].insert(0, {"id": "merge", "options": {}}) +y += 8 + dash = { "uid": "ipx", "title": "iPX", "tags": ["ipx"], "timezone": "browser", "schemaVersion": 39, "time": {"from": "now-24h", "to": "now"}, "refresh": "1m", "editable": True, diff --git a/playwright.config.ts b/playwright.config.ts index 7c7cc83..69c1af6 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,6 +4,12 @@ import * as setup from './tests/ui/global-setup.ts'; // Before anything else, including the servers below. setup.prepare(); +// Measuring the API (#394): an instrumented binary writes its coverage only as it exits, and +// Playwright otherwise kills its servers outright. Asked to stop, both shut down on their own. +// Where they write it, as coverage.sh sets it; passed on to each server whatever Playwright passes. +const PROFILE: Record = process.env.LLVM_PROFILE_FILE ? { LLVM_PROFILE_FILE: process.env.LLVM_PROFILE_FILE } : {}; +const STOP = setup.API_COVERAGE ? { gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 15_000 } } : {}; + // Real browser against a real daemon. The stub-DOM smoke test catches a script that // fails to load; it cannot catch a wrong selector, a handler that runs but does nothing, // or a page that renders empty -- which is exactly what has slipped through before. @@ -34,12 +40,13 @@ export default defineConfig({ }, { // The directory the Directory shows (#205), on SQLite in the scratch directory. - command: `cargo build -q -p ipx-directory && exec ./target/debug/ipx-directory --config ${setup.root}/directory.toml`, + command: `${setup.API_COVERAGE ? '' : 'cargo build -q -p ipx-directory && '}exec ${setup.DIRECTORY_BIN} --config ${setup.root}/directory.toml`, url: `${setup.DIRECTORY}/healthz`, reuseExistingServer: false, timeout: 180_000, stdout: 'ignore', - env: { DIRECTORY_DATABASE_URL: `sqlite://${setup.root}/directory.db?mode=rwc` }, + env: { DIRECTORY_DATABASE_URL: `sqlite://${setup.root}/directory.db?mode=rwc`, ...PROFILE }, + ...STOP, }, { // The frontend as production serves it (#203): built from web/, served by Caddy on the @@ -52,8 +59,9 @@ export default defineConfig({ env: { IPX_COVERAGE: process.env.IPX_COVERAGE ?? '', WEB_BIND: '127.0.0.1:8791', WEB_ROOT: `${setup.root}/web`, IPX_UPSTREAM: '127.0.0.1:8796', HOME: setup.root, XDG_DATA_HOME: setup.root, XDG_CONFIG_HOME: setup.root }, }, { - // Build first so the tests always run against current source. - command: 'cargo build -q && exec ./target/debug/ipx daemon', + // Build first so the tests always run against current source, unless the instrumented + // build is what is measured (#394). + command: `${setup.API_COVERAGE ? '' : 'cargo build -q && '}exec ${setup.IPX_BIN} daemon`, port: 8796, reuseExistingServer: false, timeout: 180_000, @@ -70,7 +78,9 @@ export default defineConfig({ // Without one, the tests of the server paying for Signals see that it cannot. TYPESAFE_KEY: process.env.TYPESAFE_KEY ?? '', IPX_TYPESAFE_URL: 'http://127.0.0.1:8792/jev', + ...PROFILE, }, + ...STOP, }, ], }); diff --git a/scripts/coverage-metrics.ts b/scripts/coverage-metrics.ts new file mode 100644 index 0000000..8d0bb85 --- /dev/null +++ b/scripts/coverage-metrics.ts @@ -0,0 +1,72 @@ +// The last coverage run of each suite, for Grafana (#394): lines the tests reach and lines there +// are, by suite and part, as a file node-exporter's textfile collector serves to Prometheus. +// +// node scripts/coverage-metrics.ts (scripts/coverage.sh runs it after each report) +// +// Suites: `cargo` (the cargo tests, target/coverage/cargo), `browser` (the browser suite against an +// instrumented ipx and ipx-directory, target/coverage/api) and `page` (web/src under the browser +// suite, target/coverage/page). Parts: the crate, or `page`. Each run writes every suite that has +// a report, so one suite's run leaves the others' numbers as they were. +import fs from 'node:fs'; +import path from 'node:path'; + +const COVERAGE = path.join(import.meta.dirname, '..', 'target', 'coverage'); +// node-exporter reads it as /host/... (it mounts / at /host), on /mnt/fast beside its appdata. +const OUT = process.env.IPX_COVERAGE_TEXTFILE ?? '/mnt/fast/appdata/node-exporter/textfile'; + +type Count = { covered: number, total: number }; +type Suite = { parts: Map, at: number }; + +/// The crate a path in cargo-llvm-cov's summary belongs to: directory/src/... is ipx-directory's. +const crateOf = (file: string) => file.startsWith('directory/') ? 'ipx-directory' : file.startsWith('feed/') ? 'ipx-feed' : 'ipx'; + +function add(parts: Map, part: string, covered: number, total: number) { + const c = parts.get(part) ?? { covered: 0, total: 0 }; + c.covered += covered; c.total += total; + parts.set(part, c); +} + +/// cargo-llvm-cov's summary: a line a file, Lines and Missed Lines its 8th and 9th columns. +function rust(dir: string): Suite | null { + const file = path.join(COVERAGE, dir, 'summary.txt'); + if (!fs.existsSync(file)) return null; + const parts = new Map(); + for (const line of fs.readFileSync(file, 'utf8').split('\n')) { + const cols = line.trim().split(/\s+/); + if (cols.length < 10 || !cols[0]!.endsWith('.rs')) continue; + const lines = Number(cols[7]), missed = Number(cols[8]); + if (Number.isFinite(lines) && Number.isFinite(missed)) add(parts, crateOf(cols[0]!), lines - missed, lines); + } + return { parts, at: fs.statSync(file).mtimeMs / 1000 }; +} + +function page(): Suite | null { + const file = path.join(COVERAGE, 'page', 'summary.json'); + if (!fs.existsSync(file)) return null; + const { lines } = JSON.parse(fs.readFileSync(file, 'utf8')) as { lines: ({ file: string } & Count)[] }; + const parts = new Map(); + for (const l of lines) add(parts, 'page', l.covered, l.total); + return { parts, at: fs.statSync(file).mtimeMs / 1000 }; +} + +const suites: [string, Suite | null][] = [['cargo', rust('cargo')], ['browser', rust('api')], ['page', page()]]; +const out = [ + '# HELP ipx_coverage_lines Lines of iPX the tests reached and lines there are, as each suite\'s last coverage run found them (#394).', + '# TYPE ipx_coverage_lines gauge', +]; +for (const [suite, s] of suites) if (s) for (const [part, c] of s.parts) { + out.push(`ipx_coverage_lines{suite="${suite}",part="${part}",state="covered"} ${c.covered}`); + out.push(`ipx_coverage_lines{suite="${suite}",part="${part}",state="total"} ${c.total}`); +} +out.push('# HELP ipx_coverage_run_timestamp_seconds When each suite\'s last coverage report was made.'); +out.push('# TYPE ipx_coverage_run_timestamp_seconds gauge'); +for (const [suite, s] of suites) if (s) out.push(`ipx_coverage_run_timestamp_seconds{suite="${suite}"} ${Math.round(s.at)}`); + +fs.mkdirSync(OUT, { recursive: true }); +// Written aside and renamed in, so a scrape never reads half a file. +const file = path.join(OUT, 'ipx_coverage.prom'); +fs.writeFileSync(file + '.tmp', out.join('\n') + '\n'); +fs.renameSync(file + '.tmp', file); +console.log(`coverage for Grafana: ${file}`); +for (const [suite, s] of suites) if (s) for (const [part, c] of s.parts) + console.log(` ${suite} ${part}: ${(100 * c.covered / c.total).toFixed(1)}% of ${c.total} lines`); diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 14c6dd7..5569ef6 100644 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -1,17 +1,22 @@ #!/bin/sh -# Which lines of the Rust the tests reach (#390), inside the limits builds are held to. +# Which lines of the Rust the tests reach (#390, #394), inside the limits builds are held to. # -# sh scripts/coverage.sh all three crates, then the report -# sh scripts/coverage.sh ipx-feed one crate's tests, then the report of what has run +# sh scripts/coverage.sh the cargo tests, all three crates, then their report +# sh scripts/coverage.sh ipx-feed one crate's cargo tests, then the report of what has run +# sh scripts/coverage.sh api the browser suite against an instrumented ipx and +# ipx-directory: what the API and the daemon run (and the +# page's coverage, which the same run measures) # # A coverage build instruments every crate, dependencies included, in a target of its own: a whole -# rebuild, the kind that took Tower down on 2026-11-04. So it runs in a container under the same +# rebuild, the kind that took Tower down on 2026-11-04. So it builds in a container under the same # cap as scripts/build-image.sh's builder (6 GB, six threads), past which the kernel kills it and # production is left alone; and, as CLAUDE.md asks of tests, one crate at a time. If the cap kills -# it, say so and leave the cap be. +# it, say so and leave the cap be. The browser suite runs here, as it always does, its servers the +# container's build: the image is Ubuntu's, as code-server is, so they start here. # -# The report: target/coverage/summary.txt (a line a file), target/coverage/lcov.info, and -# target/coverage/html/index.html. +# Reports: target/coverage/cargo and target/coverage/api (summary.txt, a line a file; lcov.info; +# html/index.html), target/coverage/page (npm run coverage:page's, or api's). Each run ends by +# writing the numbers for Grafana (scripts/coverage-metrics.ts). set -eu MEM=6g @@ -21,13 +26,14 @@ MAX_LOAD=12 IMAGE=192.168.1.130:5000/ipx-coverage:latest # The Docker daemon is Tower's, so its paths are Tower's: this tree is /mnt/user/vscode/ipx there, # and target/ is a link to target-real on /mnt/fast (builds through /mnt/user's FUSE took the load -# to 248, CLAUDE.md). +# to 248, CLAUDE.md). code-server sees /mnt/fast at the same path, so the instrumented binaries +# and the profiles they write are at the same place for both. SRC=/mnt/user/vscode/ipx TARGET=/mnt/fast/vscode/ipx/target-real REGISTRY=$TARGET/coverage-registry +LLVM_TARGET=$TARGET/llvm-cov-target cd "$(dirname "$0")/.." -crates=${1:-ipx-feed ipx-directory ipx} room() { avail=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo) @@ -56,8 +62,45 @@ run() { } } +# The report of the profiles in the target, into target/coverage/$1, with the tree's own paths. +report() { + room + mkdir -p "$TARGET/coverage/$1" + run sh -c "cargo llvm-cov report --lcov --output-path /target/coverage/$1/lcov.info \ + && cargo llvm-cov report --html --output-dir /target/coverage/$1 \ + && cargo llvm-cov report --summary-only > /target/coverage/$1/summary.txt" + # The paths inside the container are /work/...; the tree's own, from here. + sed -i 's#/work/##' "$TARGET/coverage/$1/summary.txt" "$TARGET/coverage/$1/lcov.info" + tail -1 "$TARGET/coverage/$1/summary.txt" +} + docker pull -q "$IMAGE" >/dev/null mkdir -p "$REGISTRY" + +if [ "${1:-}" = api ]; then + # Built as cargo-llvm-cov builds for a test of its own, instrumented, with no tests: the + # binaries the browser suite starts. Profiles from an earlier run are not this one's. + run cargo llvm-cov clean --workspace + for p in ipx-directory ipx; do + room + echo "== build $p" + run sh -c "eval \"\$(cargo llvm-cov show-env --export-prefix)\" && cargo build --locked -p $p" + done + room + echo "== the browser suite" + # Not `set -e`'s to stop on: a failing test still ran what it ran, and the report says so; the + # suite's own exit code is this script's at the end. + suite=0 + IPX_API_COVERAGE=$LLVM_TARGET IPX_COVERAGE=1 LLVM_PROFILE_FILE="$LLVM_TARGET/ipx-browser-%p-%14m.profraw" \ + npx playwright test || suite=$? + ls "$LLVM_TARGET"/ipx-browser-*.profraw >/dev/null 2>&1 \ + || { echo "no profiles were written: did the servers stop on SIGTERM?" >&2; exit 1; } + report api + node scripts/coverage-metrics.ts + exit "$suite" +fi + +crates=${1:-ipx-feed ipx-directory ipx} # A whole run starts from nothing; one crate adds to what has run. [ $# -eq 0 ] && run cargo llvm-cov clean --workspace for c in $crates; do @@ -65,11 +108,5 @@ for c in $crates; do echo "== $c" run cargo llvm-cov --no-report --locked -p "$c" done -room -mkdir -p "$TARGET/coverage" -run sh -c 'cargo llvm-cov report --lcov --output-path /target/coverage/lcov.info \ - && cargo llvm-cov report --html --output-dir /target/coverage \ - && cargo llvm-cov report --summary-only > /target/coverage/summary.txt' -# The paths inside the container are /work/...; the tree's own, from here. -sed -i 's#/work/##' "$TARGET/coverage/summary.txt" "$TARGET/coverage/lcov.info" -tail -1 "$TARGET/coverage/summary.txt" +report cargo +node scripts/coverage-metrics.ts diff --git a/tests/ui/app.spec.ts b/tests/ui/app.spec.ts index 2a060b6..61a1dba 100644 --- a/tests/ui/app.spec.ts +++ b/tests/ui/app.spec.ts @@ -773,7 +773,7 @@ test('a second person has their own feeds and their own read state', async ({ br IPX_DATA_DIR: `${setup.root}/data`, }; try { - execFileSync('./target/debug/ipx', ['user', 'add', 'sam'], { input: 'sampassword', env }); + execFileSync(setup.IPX_BIN, ['user', 'add', 'sam'], { input: 'sampassword', env }); } catch (e) { if (!said(e).includes('already exists')) throw e; } @@ -979,7 +979,7 @@ test('an export from one account imports into another', async ({ page, browser } IPX_DATA_DIR: `${setup.root}/data`, }; try { - execFileSync('./target/debug/ipx', ['user', 'add', 'opal'], { input: 'opalpassword', env }); + execFileSync(setup.IPX_BIN, ['user', 'add', 'opal'], { input: 'opalpassword', env }); } catch (e) { if (!said(e).includes('already exists')) throw e; } @@ -1021,7 +1021,7 @@ test('ipx import subscribes the admin, and ipx export writes the feeds out', asy fs.writeFileSync(path.join(dir, 'config.toml'), `[general]\ndownload_dir = "${dir}/downloads"\nsocket = "${dir}/ipx.sock"\n`); const env = { ...process.env, IPX_CONFIG: path.join(dir, 'config.toml'), IPX_DATA_DIR: path.join(dir, 'data') }; - const ipx = (args: string[], input?: string) => execFileSync('./target/debug/ipx', args, { env, input, encoding: 'utf8' }); + const ipx = (args: string[], input?: string) => execFileSync(setup.IPX_BIN, args, { env, input, encoding: 'utf8' }); ipx(['user', 'add', 'boss'], 'bosspassword'); // the first account is the admin const opml = path.join(dir, 'in.opml'); @@ -1045,7 +1045,7 @@ test('the Directory lists what servers share, the most subscribed first, but nev IPX_DATA_DIR: `${setup.root}/data`, }; try { - execFileSync('./target/debug/ipx', ['user', 'add', 'piper'], { input: 'piperpassword', env }); + execFileSync(setup.IPX_BIN, ['user', 'add', 'piper'], { input: 'piperpassword', env }); } catch (e) { if (!said(e).includes('already exists')) throw e; } @@ -1218,7 +1218,7 @@ test('the feed list keeps the icons it has drawn when a feed\'s new row comes in test('ipx add and rm while the daemon runs reach it, and its next change keeps them (#120)', async ({ page }) => { const env = { ...process.env, IPX_CONFIG: `${setup.root}/config/config.toml`, IPX_DATA_DIR: `${setup.root}/data` }; - const ipx = (...args: string[]) => execFileSync('./target/debug/ipx', args, { env, encoding: 'utf8' }); + const ipx = (...args: string[]) => execFileSync(setup.IPX_BIN, args, { env, encoding: 'utf8' }); // By title: the directory names a feed by its address, not by its id here (#205). const listed = async () => (await page.evaluate((): Promise => api('/api/directory'))).map(p => p.title); // A change of the daemon's own: it writes the whole catalogue from the copy it holds, which @@ -1598,7 +1598,7 @@ test('an admin removes a feed from the directory, and unblocks it (#206)', async // Someone who is not an admin is neither offered it nor allowed it. const env = { ...process.env, IPX_CONFIG: `${setup.root}/config/config.toml`, IPX_DATA_DIR: `${setup.root}/data` }; try { - execFileSync('./target/debug/ipx', ['user', 'add', 'piper'], { input: 'piperpassword', env }); + execFileSync(setup.IPX_BIN, ['user', 'add', 'piper'], { input: 'piperpassword', env }); } catch (e) { if (!said(e).includes('already exists')) throw e; } @@ -2201,7 +2201,7 @@ test('DeadAir is off until turned on for a show, then plays it shortened, with y // file on disk, a playlist each, and neither kept by a cache to be handed to the other. { const env = { ...process.env, IPX_CONFIG: `${setup.root}/config/config.toml`, IPX_DATA_DIR: `${setup.root}/data` }; - try { execFileSync('./target/debug/ipx', ['user', 'add', 'quinn'], { input: 'quinnpassword', env }); } + try { execFileSync(setup.IPX_BIN, ['user', 'add', 'quinn'], { input: 'quinnpassword', env }); } catch (e) { if (!said(e).includes('already exists')) throw e; } const ctx = await page.context().browser()!.newContext(); const quinn = await ctx.newPage(); diff --git a/tests/ui/coverage-teardown.ts b/tests/ui/coverage-teardown.ts index 76cb897..10f3fa9 100644 --- a/tests/ui/coverage-teardown.ts +++ b/tests/ui/coverage-teardown.ts @@ -1,7 +1,14 @@ -// The page's coverage report, once every test has added its page's (coverage.ts, #390). +// The page's coverage report, once every test has added its page's (coverage.ts, #390), and its +// lines file by file in summary.json, which scripts/coverage-metrics.ts reads for Grafana (#394). +import fs from 'node:fs'; +import path from 'node:path'; import { CoverageReport } from 'monocart-coverage-reports'; import { coverageOptions } from './coverage.ts'; export default async function teardown() { - if (process.env.IPX_COVERAGE) await new CoverageReport(coverageOptions).generate(); + if (!process.env.IPX_COVERAGE) return; + const results = await new CoverageReport(coverageOptions).generate(); + if (!results) return; + const lines = results.files.map((f) => ({ file: f.sourcePath, covered: f.summary.lines.covered, total: f.summary.lines.total })); + fs.writeFileSync(path.join(coverageOptions.outputDir!, 'summary.json'), JSON.stringify({ lines }, null, 2)); } diff --git a/tests/ui/global-setup.ts b/tests/ui/global-setup.ts index eca2476..25e6848 100644 --- a/tests/ui/global-setup.ts +++ b/tests/ui/global-setup.ts @@ -88,4 +88,11 @@ token_sha256 = "${crypto.createHash('sha256').update(DIRECTORY_TOKEN).digest('he `); } -export { prepare, root, TOKEN, DIRECTORY, DIRECTORY_TOKEN }; +/// With IPX_API_COVERAGE set to an instrumented build's target (`sh scripts/coverage.sh api`, #394), the +/// suite runs that build's ipx and ipx-directory, which write their coverage profiles there, the +/// tests' own `ipx` commands included; without it, the debug build it makes itself. +const API_COVERAGE = process.env.IPX_API_COVERAGE ?? ''; +const IPX_BIN = API_COVERAGE ? path.join(API_COVERAGE, 'debug', 'ipx') : './target/debug/ipx'; +const DIRECTORY_BIN = API_COVERAGE ? path.join(API_COVERAGE, 'debug', 'ipx-directory') : './target/debug/ipx-directory'; + +export { prepare, root, TOKEN, DIRECTORY, DIRECTORY_TOKEN, API_COVERAGE, IPX_BIN, DIRECTORY_BIN }; diff --git a/tsconfig.tools.json b/tsconfig.tools.json index 578a093..2148af2 100644 --- a/tsconfig.tools.json +++ b/tsconfig.tools.json @@ -15,5 +15,5 @@ "skipLibCheck": true, "types": ["node"] }, - "include": ["target/page-types/web/src/*.d.ts", "tests/*.ts", "tests/ui/**/*.ts", "tests/load/run.ts", "web/*.mts", "site/*.mts", "playwright.config.ts"] + "include": ["target/page-types/web/src/*.d.ts", "scripts/*.ts", "tests/*.ts", "tests/ui/**/*.ts", "tests/load/run.ts", "web/*.mts", "site/*.mts", "playwright.config.ts"] }