diff --git a/CHANGELOG.md b/CHANGELOG.md index cc8d504..a795830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A feed whose address answers with nothing, as a lapsed domain does behind a DNS filter's block page, shows as failing, with what to do about it, instead of as a show that has not posted yet. - The search box finds episodes in Currently Listening. It did nothing there. - `ipx add`, `ipx rm` and `ipx import`, run while the daemon runs, reach it at once. It kept its old list of feeds and wrote it back at its next change, undoing them. +- On SQLite, several requests are answered at once instead of one at a time. With 25 people browsing, a page of a feed's items comes back in 10ms rather than 130. - On Postgres, a write no longer waits for the database server's disk. A scan read about one feed a second and now reads 1,500 in 20 seconds. - Checking every feed no longer makes every icon in the feed list flash while it runs: the list keeps the icons it has already drawn. diff --git a/CLAUDE.md b/CLAUDE.md index 32c657f..14aacff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -161,9 +161,9 @@ seeking through files, a flood of wrong passwords. Things about them: * The daemon is on **SQLite** unless `--postgres`, which puts it in `IPX_TEST_DATABASE_URL`, in a schema of its own (`ipx_load`) dropped and made again each run. That URL, in `/src/.envrc`, is `ipodderx_test` on the `databases` project's server, production's, as `ipodderx`, taken from - `ipx.env`; the harness refuses a database whose name does not end in `_test`. SQLite's numbers - carry its single connection (#136); Postgres's are what production would see, and a run loads - the server production's database is on, so not while someone is using iPX in earnest. + `ipx.env`; the harness refuses a database whose name does not end in `_test`. Postgres's + numbers are what production would see, and a run loads the server production's database is on, + so not while someone is using iPX in earnest. Each budget has a value for each database. * The Rust tests run on Postgres with the same URL: `. /src/.envrc && cargo test`. Each test makes a schema `ipxt__`, and the next run drops those an earlier one left. diff --git a/src/db.rs b/src/db.rs index 4fe15b2..e13946d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -131,6 +131,15 @@ async fn connect(location: &str) -> Result { // commits, about the last 0.6s, and never corrupts anything. Added to whatever options the // URL sets, so it holds for the tests' search_path too. opts.map_sqlx_postgres_opts(|o| o.options([("synchronous_commit", "off")])); + // Several connections, where SeaORM gives SQLite one unless told: every request, every scan + // write and every read went through it in turn, and twenty-five people browsing waited in + // line for it at 50 requests a second (#136). In WAL readers run beside the one writer. Each + // transaction here writes first, so it takes the write lock or waits out the busy timeout for + // it; SQLite refuses at once only a transaction that read a snapshot a write has since moved + // past. Postgres keeps sqlx's ten. + if !is_postgres(location) { + opts.max_connections(8); + } sea_orm::Database::connect(opts) .await .with_context(|| format!("opening {}", redact(location))) @@ -319,6 +328,8 @@ impl Db { } let path = std::env::temp_dir().join(format!("ipx-test-{pid}-{n}.db")); let orm = connect(&path.display().to_string()).await?; + // As Db::open has it, so the tests' connections share the file as the daemon's do. + orm.execute_unprepared("PRAGMA journal_mode = WAL").await?; create_missing(&orm).await?; Ok(Self { orm, tmp: Some(path) }) } @@ -2111,6 +2122,27 @@ pub fn now() -> i64 { mod tests { use super::*; + #[tokio::test] + async fn a_read_does_not_wait_for_a_write_in_progress() { + use sea_orm::TransactionTrait; + let db = Db::memory().await.unwrap(); + // A write held open, as a scan's or an import's is while it runs. + let tx = db.orm.begin().await.unwrap(); + tx.execute_unprepared("INSERT INTO feeds (id, url) VALUES ('held', 'http://x/held')").await.unwrap(); + // With one connection, which SQLite had (#136), this waited for the transaction to end. + let read = tokio::time::timeout(std::time::Duration::from_secs(2), db.feed_summary("other")).await; + assert!(read.is_ok(), "a read waited behind a write"); + tx.commit().await.unwrap(); + // And writers at once all get their turn, not a refusal. + let writes = (0..16).map(|i| { + let db = &db; + async move { db.record_feed(&format!("f{i}"), "http://x/f", Some("F"), None, None, None, None, None, None).await } + }); + for r in futures_util::future::join_all(writes).await { + r.unwrap(); + } + } + #[tokio::test] async fn a_commit_does_not_wait_for_the_disk() { let db = Db::memory().await.unwrap(); diff --git a/tests/load/browse.js b/tests/load/browse.js index 63bde25..d34d253 100644 --- a/tests/load/browse.js +++ b/tests/load/browse.js @@ -6,7 +6,7 @@ import { BASE, FEEDS, ITEMS, as, me, pick, feedId, p95 } from './lib.js'; // a feed, a search, the Directory and a feed's page in it. Each answer's budget is well above what // it takes now, so it fails on a query that has started asking once per feed -- the Directory // asked the database three questions a feed until 0.10.0 -- not on a slow minute. Fifty measured -// the queue for SQLite's one connection (#136) more than any query. +// the queue for what was SQLite's one connection (#136) more than any query. export const options = { scenarios: { evening: { @@ -17,14 +17,14 @@ export const options = { thresholds: { http_req_failed: ['rate==0'], checks: ['rate==1'], - // About twice each one's p95 on 2026-10-05 on Tower. SQLite: 236, 118, 133, 119, 259, 312ms; - // Postgres: 54, 34, 24, 44, 71, 88ms. - 'http_req_duration{name:feeds}': p95(500, 150), - 'http_req_duration{name:all-entries}': p95(300, 100), - 'http_req_duration{name:feed-entries}': p95(300, 100), - 'http_req_duration{name:search}': p95(300, 100), - 'http_req_duration{name:directory}': p95(600, 200), - 'http_req_duration{name:listed}': p95(700, 200), + // About twice each one's p95 on 2026-10-05 on Tower, and no less than 50ms, where a few ms + // either way is noise. SQLite: 89, 17, 10, 22, 112, 126ms; Postgres: 54, 34, 24, 44, 71, 88ms. + 'http_req_duration{name:feeds}': p95(200, 150), + 'http_req_duration{name:all-entries}': p95(50, 100), + 'http_req_duration{name:feed-entries}': p95(50, 100), + 'http_req_duration{name:search}': p95(60, 100), + 'http_req_duration{name:directory}': p95(250, 200), + 'http_req_duration{name:listed}': p95(300, 200), }, }; diff --git a/tests/load/lib.js b/tests/load/lib.js index 417ce2e..6e8b6ad 100644 --- a/tests/load/lib.js +++ b/tests/load/lib.js @@ -13,7 +13,7 @@ export const as = (name, tag) => ({ /// The listener this virtual user is, one of the hundred run.js seeded, thirty feeds each. export const me = () => `load-${(__VU - 1) % USERS}`; export const pick = a => a[Math.floor(Math.random() * a.length)]; -/// A p95 budget, in ms, for whichever database the daemon is on (run.js --postgres). Postgres's -/// are tighter: SQLite's carry the queue for its one connection (#136). +/// A p95 budget, in ms, for whichever database the daemon is on (run.js --postgres): SQLite reads +/// a file beside the daemon, Postgres answers over the network, and each is quicker at something. export const p95 = (sqlite, postgres) => [`p(95)<${__ENV.DB === 'postgres' ? postgres : sqlite}`]; export const feedId = n => `gen-${String(n).padStart(4, '0')}`; diff --git a/tests/load/listening.js b/tests/load/listening.js index e66026a..ff62b5b 100644 --- a/tests/load/listening.js +++ b/tests/load/listening.js @@ -15,12 +15,11 @@ export const options = { thresholds: { http_req_failed: ['rate==0'], checks: ['rate==1'], - // About twice each one's p95 on 2026-10-05 on Tower. SQLite: 53, 786 and 383ms; Postgres: 9, - // 225 and 153ms. Marking read answers with the feed's row, counts and all, and on SQLite - // waits behind the scans' writes for its one connection (#136), hence its budget. - 'http_req_duration{name:position}': p95(300, 100), - 'http_req_duration{name:flags}': p95(1500, 500), - 'http_req_duration{name:read-back}': p95(800, 400), + // About twice each one's p95 on 2026-10-05 on Tower. SQLite: 21, 277 and 134ms; Postgres: 9, + // 225 and 153ms. Marking read answers with the feed's row, counts and all, hence its budget. + 'http_req_duration{name:position}': p95(100, 100), + 'http_req_duration{name:flags}': p95(600, 500), + 'http_req_duration{name:read-back}': p95(300, 400), }, }; diff --git a/tests/load/run.js b/tests/load/run.js index cfa20c7..edc6c35 100644 --- a/tests/load/run.js +++ b/tests/load/run.js @@ -13,8 +13,8 @@ // // The daemon is on SQLite unless --postgres, which puts it in IPX_TEST_DATABASE_URL, the // database the Rust tests use on Postgres too (ipodderx_test on the server production's is on, in -// /src/.envrc), in a schema of its own, ipx_load, made afresh each run (#139). SQLite's numbers -// carry its one connection (#136); Postgres's are what production would see. +// /src/.envrc), in a schema of its own, ipx_load, made afresh each run (#139). Postgres's numbers +// are what production would see. // IPX_LOAD_FEEDS sets the catalogue's size (1500, near production's), IPX_LOAD_USERS the // listeners (100), IPX_LOAD_LOG=1 shows the daemon's log. const http = require('http');