From 367f431e1fcc8660e853250b6272c390dae6fa52 Mon Sep 17 00:00:00 2001 From: rays Date: Mon, 5 Oct 2026 17:09:29 +0000 Subject: [PATCH] Sync SQLite at checkpoints, not after every write (#135) The load tests (#133) found a daemon on SQLite scanning 1.3s a feed of 20 items served from the same machine: 1,500 feeds would have taken about 50 minutes. On Tower's /tmp, a loop device, one synchronous 4 KB write takes 52ms, and SQLite was in WAL mode with its default synchronous=FULL, which waits for the disk on every commit; nearly every statement is a commit of its own, a feed's entries and enclosures being written a row at a time. Subscribing the first admin to an imported catalogue of 1,500 feeds took 39s, before the web server listened. Every SQLite connection now sets synchronous=NORMAL, SQLite's own advice for WAL: it syncs at checkpoints. A crash of ipx loses nothing; a power cut can lose the last transactions, and never corrupts the file. The same scan takes 27s and the same subscribing 0.75s. Postgres, which production runs on, is unaffected. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + src/db.rs | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c71c4..ac66eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A database on SQLite, the default, no longer waits for the disk after nearly every write. On a slow disk a scan of 1,500 feeds took most of an hour and now takes half a minute. Postgres is unaffected. - The search box finds a feed in the Directory by name. It did nothing there. - Feeds that file themselves under Apple's Football or Soccer show under Sports, and those under the old Software How-To under Technology, instead of as categories of their own. - The download queue, in `ipx status`, `/api/status` and the dashboard, counts only what iPX will download on its own: older episodes beyond a show's limit, and files of feeds that do not download automatically, are listed but no longer counted as waiting. diff --git a/src/db.rs b/src/db.rs index cd895d4..0d38cd5 100644 --- a/src/db.rs +++ b/src/db.rs @@ -114,11 +114,17 @@ fn is_postgres(location: &str) -> bool { location.starts_with("postgres://") || location.starts_with("postgresql://") } -/// sqlx's defaults for SQLite are what ipx wants: foreign keys on, and a five-second wait for a -/// lock, which is what rusqlite was set to. +/// sqlx's defaults for SQLite are what ipx wants, foreign keys on and a five-second wait for a +/// lock, which is what rusqlite was set to, but for one. async fn connect(location: &str) -> Result { let mut opts = sea_orm::ConnectOptions::new(url_for(location)); opts.sqlx_logging(false); + // NORMAL, SQLite's own advice for WAL: synced at checkpoints, not after every commit. FULL, + // the default, waited for the disk after nearly every statement, and a feed's items are + // written a row at a time: 1.3s a feed on Tower's /tmp, where one synchronous write takes + // 52ms, and 39s to subscribe an admin to 1,500 feeds (#135). A crash of ipx loses nothing; + // only a power cut can lose the last transactions, and it never corrupts the file. + opts.map_sqlx_sqlite_opts(|o| o.synchronous(sea_orm::sqlx::sqlite::SqliteSynchronous::Normal)); sea_orm::Database::connect(opts) .await .with_context(|| format!("opening {}", redact(location))) @@ -2099,6 +2105,16 @@ pub fn now() -> i64 { mod tests { use super::*; + #[tokio::test] + async fn sqlite_syncs_at_checkpoints_not_every_commit() { + let db = Db::memory().await.unwrap(); + if db.orm.get_database_backend() != sea_orm::DbBackend::Sqlite { + return; + } + let row = db.orm.query_one_raw(Statement::from_string(sea_orm::DbBackend::Sqlite, "PRAGMA synchronous")).await.unwrap(); + assert_eq!(row.unwrap().try_get_by_index::(0).unwrap(), 1, "NORMAL, on every connection in the pool (#135)"); + } + #[tokio::test] async fn a_database_from_before_descriptions_gains_the_column_and_reads_each_feed_whole() { let db = Db::memory().await.unwrap();