Run the load tests on Postgres too, as production runs (#139)

node tests/load/run.js --postgres puts the scratch daemon in IPX_TEST_DATABASE_URL, the database
the Rust tests already use on Postgres, ipodderx_test on the databases project's server, in a
schema of its own, ipx_load, dropped and made again each run as /tmp/ipx-load is wiped. The URL
is in /src/.envrc, taken from ipx.env. It is production's server and role, so the harness refuses
a database whose name does not end in _test, and hands psql the password in its environment,
where no other process on Tower can read it. psql comes from /src/install.sh.

After seeding it runs ANALYZE: a schema filled seconds before has no planner statistics until
autovacuum gets there, which it did partway through a test, and the search's p95 swung between
40ms and 166ms from run to run. Analyzed first, three runs gave 37-38ms.

Each budget now has a value for each database, about twice what it measures on Tower: Postgres
answers the feed list at p95 54ms to SQLite's 242, having no one connection to queue for (#136).
Running it found #140.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CLAUDE.md

@@ -158,8 +158,14 @@ seeking through files, a flood of wrong passwords. Things about them:
 * `ipx status`, the Docker healthcheck, runs every second throughout, and a run fails if one takes
   the healthcheck's 5s.
 * One at a time: `node tests/load/run.js signin`. All four take about six minutes.
-* The daemon is on **SQLite**, so its numbers carry SQLite's single connection (#136); production
+* The daemon is on **SQLite** unless `--postgres`, which puts it in `IPX_TEST_DATABASE_URL`, in
-  is on Postgres, which the harness cannot run on yet (#139).
+  a schema of its own (`ipx_load`) dropped and made again each run. That URL, in `/src/.envrc`, is
+  `ipodderx_test` on the `databases` project's server, production's, as `ipodderx`, taken from
+  `ipx.env`; the harness refuses a database whose name does not end in `_test`. SQLite's numbers
+  carry its single connection (#136); Postgres's are what production would see, and a run loads
+  the server production's database is on, so not while someone is using iPX in earnest.
+* The Rust tests run on Postgres with the same URL: `. /src/.envrc && cargo test`. Each test
+  makes a schema `ipxt_<pid>_<n>`, and the next run drops those an earlier one left.
 
 Non-trivial logic leaves one runnable check behind. Pure functions (`merge_policy`, `pick`,
 `matches_keywords`, `parse_interval`) are the easiest place to put it.

tests/load/browse.js

@@ -1,6 +1,6 @@
 import http from 'k6/http';
 import { check, sleep } from 'k6';
-import { BASE, FEEDS, ITEMS, as, me, pick, feedId } from './lib.js';
+import { BASE, FEEDS, ITEMS, as, me, pick, feedId, p95 } from './lib.js';
 
 // An evening's browsing: twenty-five people at once opening their feed list, All Subscriptions,
 // a feed, a search, the Directory and a feed's page in it. Each answer's budget is well above what
@@ -17,13 +17,14 @@ export const options = {
   thresholds: {
     http_req_failed: ['rate==0'],
     checks: ['rate==1'],
-    // About twice each one's p95 on 2026-10-05, SQLite on Tower: 236, 118, 133, 119, 259, 312ms.
+    // About twice each one's p95 on 2026-10-05 on Tower. SQLite: 236, 118, 133, 119, 259, 312ms;
-    'http_req_duration{name:feeds}': ['p(95)<500'],
+    // Postgres: 54, 34, 24, 44, 71, 88ms.
-    'http_req_duration{name:all-entries}': ['p(95)<300'],
+    'http_req_duration{name:feeds}': p95(500, 150),
-    'http_req_duration{name:feed-entries}': ['p(95)<300'],
+    'http_req_duration{name:all-entries}': p95(300, 100),
-    'http_req_duration{name:search}': ['p(95)<300'],
+    'http_req_duration{name:feed-entries}': p95(300, 100),
-    'http_req_duration{name:directory}': ['p(95)<600'],
+    'http_req_duration{name:search}': p95(300, 100),
-    'http_req_duration{name:listed}': ['p(95)<700'],
+    'http_req_duration{name:directory}': p95(600, 200),
+    'http_req_duration{name:listed}': p95(700, 200),
   },
 };
 

tests/load/lib.js

@@ -13,4 +13,7 @@ export const as = (name, tag) => ({
 /// The listener this virtual user is, one of the hundred run.js seeded, thirty feeds each.
 export const me = () => `load-${(__VU - 1) % USERS}`;
 export const pick = a => a[Math.floor(Math.random() * a.length)];
+/// A p95 budget, in ms, for whichever database the daemon is on (run.js --postgres). Postgres's
+/// are tighter: SQLite's carry the queue for its one connection (#136).
+export const p95 = (sqlite, postgres) => [`p(95)<${__ENV.DB === 'postgres' ? postgres : sqlite}`];
 export const feedId = n => `gen-${String(n).padStart(4, '0')}`;

tests/load/listening.js

@@ -1,6 +1,6 @@
 import http from 'k6/http';
 import { check, sleep } from 'k6';
-import { BASE, as, me, pick } from './lib.js';
+import { BASE, as, me, pick, p95 } from './lib.js';
 
 // Players saving where people are while scans write. A hundred listeners each save a position
 // every second -- the player saves every ten, so this is a thousand people listening -- mark an
@@ -15,12 +15,12 @@ export const options = {
   thresholds: {
     http_req_failed: ['rate==0'],
     checks: ['rate==1'],
-    // About twice each one's p95 on 2026-10-05, SQLite on Tower: 53, 786 and 383ms. Marking
+    // About twice each one's p95 on 2026-10-05 on Tower. SQLite: 53, 786 and 383ms; Postgres: 9,
-    // read answers with the feed's row, counts and all, and waits behind the scans' writes on
+    // 225 and 153ms. Marking read answers with the feed's row, counts and all, and on SQLite
-    // SQLite's one connection (#136), hence its budget.
+    // waits behind the scans' writes for its one connection (#136), hence its budget.
-    'http_req_duration{name:position}': ['p(95)<300'],
+    'http_req_duration{name:position}': p95(300, 100),
-    'http_req_duration{name:flags}': ['p(95)<1500'],
+    'http_req_duration{name:flags}': p95(1500, 500),
-    'http_req_duration{name:read-back}': ['p(95)<800'],
+    'http_req_duration{name:read-back}': p95(800, 400),
   },
 };
 

tests/load/media.js

@@ -10,7 +10,8 @@ export const options = {
   thresholds: {
     http_req_failed: ['rate==0'],
     checks: ['rate==1'],
-    // About four times its p95 on 2026-10-05, 48ms: a range is a file read, little to vary.
+    // About four times its p95 on 2026-10-05, 48ms on SQLite and 45 on Postgres: a range is a
+    // file read, little to vary.
     'http_req_duration{name:range}': ['p(95)<200'],
   },
 };

tests/load/run.js

@@ -3,7 +3,7 @@
 // database and to the healthcheck. The browser suite drives one person against a handful of
 // feeds, one request at a time.
 //
-//   node tests/load/run.js [browse|listening|media|signin ...]     default: all of them
+//   node tests/load/run.js [--postgres] [browse|listening|media|signin ...]     default: all four
 //
 // It builds a release binary (debug Argon2 alone takes a second a sign-in, which would measure
 // the build, not ipx), serves generated feeds from this process, starts a daemon on its own
@@ -11,10 +11,12 @@
 // While each runs, `ipx status` -- the Docker healthcheck -- is run every second, and the run
 // fails if any answer takes as long as the healthcheck's 5s timeout.
 //
+// The daemon is on SQLite unless --postgres, which puts it in IPX_TEST_DATABASE_URL, the
+// database the Rust tests use on Postgres too (ipodderx_test on the server production's is on, in
+// /src/.envrc), in a schema of its own, ipx_load, made afresh each run (#139). SQLite's numbers
+// carry its one connection (#136); Postgres's are what production would see.
 // IPX_LOAD_FEEDS sets the catalogue's size (1500, near production's), IPX_LOAD_USERS the
-// listeners (100), IPX_LOAD_LOG=1 shows the daemon's log. The daemon is on SQLite, not Postgres
+// listeners (100), IPX_LOAD_LOG=1 shows the daemon's log.
-// as production is: ponytail: Postgres needs a database of its own emptied before each run,
-// which this cannot do yet (#139).
 const http = require('http');
 const fs = require('fs');
 const path = require('path');
@@ -31,6 +33,9 @@ const ITEMS = 20;
 const BASE = `http://127.0.0.1:${WEB}`;
 const bin = path.join(repo, 'target/release/ipx');
 const SCRIPTS = ['browse', 'listening', 'media', 'signin'];
+const POSTGRES = process.argv.includes('--postgres');
+const PG_URL = process.env.IPX_TEST_DATABASE_URL || '';
+const PG_SCHEMA = 'ipx_load';
 const env = {
   ...process.env,
   IPX_CONFIG: `${root}/config/config.toml`,
@@ -38,7 +43,10 @@ const env = {
   IPX_LOG: 'ipx=info',
   IPX_LOG_FORMAT: 'json',
   // Nothing of the test reaches production's database or traces, or a paid API.
-  IPX_DATABASE_URL: '',
+  // In its own schema, as Db::memory puts each Rust test, and with notices off, as url_for does.
+  IPX_DATABASE_URL: POSTGRES
+    ? `${PG_URL}${PG_URL.includes('?') ? '&' : '?'}options=-c%20search_path%3D${PG_SCHEMA}%20-c%20client_min_messages%3Dwarning`
+    : '',
   OTEL_EXPORTER_OTLP_ENDPOINT: '',
   TYPESAFE_KEY: '',
 };
@@ -93,6 +101,26 @@ const feedUrl = n => `http://127.0.0.1:${GEN}/feed/${n}.xml`;
 
 // ---- the daemon -----------------------------------------------------------------------------
 
+/// The load tests' schema dropped and made again, as /tmp/ipx-load is wiped. Only in a database
+/// made for tests: the URL is production's server and role, and one slip of a name would
+/// otherwise point this at production's database.
+function freshPostgres() {
+  const u = PG_URL && new URL(PG_URL);
+  if (!u || !/_test$/.test(u.pathname)) {
+    throw new Error('--postgres needs IPX_TEST_DATABASE_URL, a database whose name ends in _test (. /src/.envrc)');
+  }
+  psql(`DROP SCHEMA IF EXISTS ${PG_SCHEMA} CASCADE`, `CREATE SCHEMA ${PG_SCHEMA}`);
+}
+
+/// Statements run in the test database, given by its URL in psql's environment, not its
+/// arguments, where every process on Tower could read the password.
+function psql(...sql) {
+  const u = new URL(PG_URL);
+  const env = { ...process.env, PGHOST: u.hostname, PGPORT: u.port || '5432', PGUSER: decodeURIComponent(u.username),
+    PGPASSWORD: decodeURIComponent(u.password), PGDATABASE: u.pathname.slice(1), PGOPTIONS: `-c search_path=${PG_SCHEMA}` };
+  execFileSync('psql', ['-q', '-v', 'ON_ERROR_STOP=1', ...sql.flatMap(s => ['-c', s])], { env, stdio: ['ignore', 'ignore', 'inherit'] });
+}
+
 function writeConfig() {
   fs.rmSync(root, { recursive: true, force: true });
   for (const d of ['config', 'data', 'downloads']) fs.mkdirSync(path.join(root, d), { recursive: true });
@@ -195,13 +223,14 @@ function watchHealth() {
 function k6(script) {
   return new Promise(resolve => {
     const run = spawn('k6', ['run', '--quiet', '-e', `BASE=${BASE}`, '-e', `USERS=${USERS}`, '-e', `FEEDS=${FEEDS}`,
+      '-e', `DB=${POSTGRES ? 'postgres' : 'sqlite'}`,
       path.join(__dirname, `${script}.js`)], { stdio: 'inherit' });
     run.on('exit', code => resolve(code));
   });
 }
 
 (async () => {
-  const only = process.argv.slice(2);
+  const only = process.argv.slice(2).filter(a => a !== '--postgres');
   for (const s of only) if (!SCRIPTS.includes(s)) { console.error(`no load test called ${s}: ${SCRIPTS.join(', ')}`); process.exit(2); }
   if (spawnSync('k6', ['version']).error) { console.error('k6 is not installed: see install.sh'); process.exit(2); }
   console.log('building the release binary...');
@@ -209,6 +238,8 @@ function k6(script) {
 
   const feeds = serveFeeds();
   writeConfig();
+  if (POSTGRES) freshPostgres();
+  console.log(`on ${POSTGRES ? 'Postgres' : 'SQLite'}`);
   // Its log kept out of this run's output, which is k6's; IPX_LOAD_LOG=1 shows it.
   const daemon = spawn(bin, ['daemon'], { env, stdio: ['ignore', process.env.IPX_LOAD_LOG ? 'inherit' : 'ignore', 'pipe'] });
   daemon.stderr.setEncoding('utf8');
@@ -221,6 +252,10 @@ function k6(script) {
     const first = await waitForLog(daemon, ev => ev.ev === 'scan_done' && ev.feeds > 0, 600_000);
     console.log(`first scan: ${first.feeds} feeds in ${((Date.now() - t) / 1000).toFixed(1)}s`);
     await seed();
+    // The planner's statistics, as production's long-standing tables have them. A schema filled
+    // seconds ago has none until autovacuum gets to it, which it did partway through a test, and
+    // the search's p95 swung from 40ms to 166ms between runs with the plan it changed.
+    if (POSTGRES) psql('ANALYZE');
     // The log is drained from here on, or the pipe fills and the daemon blocks writing to it.
     daemon.stderr.resume();
     for (const script of only.length ? only : SCRIPTS) {

tests/load/signin.js

@@ -17,7 +17,8 @@ export const options = {
   thresholds: {
     http_req_failed: ['rate==0'],
     checks: ['rate==1'],
-    // 57ms on 2026-10-05; 4.3s while password checks ran on the async workers (#137).
+    // 57ms on 2026-10-05 on SQLite, 68 on Postgres; 4.3s while password checks ran on the async
+    // workers (#137).
     'http_req_duration{name:meanwhile}': ['p(95)<300'],
     // Known name against unknown, one straight after the other: no gap but noise. 0.2ms on
     // 2026-10-05; 31ms while an unknown name was refused without a check (#138).