From e8fd3fe9ed673867a38203dd31384a1feda7dccf Mon Sep 17 00:00:00 2001 From: rays Date: Tue, 29 Sep 2026 17:24:09 +0000 Subject: [PATCH] Load the feed list in five queries, not six per feed (#94) GET /api/feeds called feed_summary, http_state, blocklist and unread_count for every feed: about 950 round trips to Postgres for 160 feeds, 320 ms on every page load. Db::feed_list asks for the feed rows, entry counts, download counts, the person's unread counts and block lists once each, and the handler reads from that. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + src/db.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/web.rs | 13 +++---- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa9026..bbb213e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The feed list loads several times faster: it was asking the database six questions per feed. - Artwork a feed offers only over plain http shows on the https site too. - A failing feed is easy to spot in any theme: a mark on its artwork, what is wrong in place of its counts, and its artwork greyed out once it has failed for a day. diff --git a/src/db.rs b/src/db.rs index 3bc7fd6..8fb54f9 100644 --- a/src/db.rs +++ b/src/db.rs @@ -333,6 +333,77 @@ impl Db { sum.downloaded = self.downloaded_count(feed_id).await?; Ok(sum) } + + /// What the feed list shows of every feed, for one person, in five queries whatever the + /// number of feeds. Asked feed by feed (feed_summary, http_state, blocklist, unread_count) + /// it was six round trips a feed, about 950 for 160 feeds and 320 ms a page load (#94). + pub async fn feed_list(&self, user_id: i64) -> Result> { + use std::collections::HashMap; + let counts = |sql: &'static str, args: Vec| async move { + self.rows(sql, args) + .await? + .iter() + .map(|r| Ok((r.try_get::("", "feed_id")?, r.try_get::("", "n")?))) + .collect::>>() + }; + let entries = counts("SELECT feed_id, count(*) AS n FROM entries GROUP BY feed_id", vec![]).await?; + let downloaded = + counts("SELECT feed_id, count(*) AS n FROM enclosures WHERE path IS NOT NULL GROUP BY feed_id", vec![]) + .await?; + let unread = counts( + "SELECT e.feed_id, count(*) AS n FROM entries e + JOIN subscriptions sub ON sub.user_id = $1 AND sub.feed_id = e.feed_id + LEFT JOIN entry_state s + ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid + WHERE NOT coalesce(s.read, false) + AND NOT EXISTS (SELECT 1 FROM hidden h + WHERE h.user_id = $1 AND h.feed_id = e.feed_id AND h.guid = e.guid) + GROUP BY e.feed_id", + vec![user_id.into()], + ) + .await?; + let mut blocked: HashMap> = blocklists::Entity::find() + .filter(blocklists::Column::UserId.eq(user_id)) + .all(&self.orm) + .await? + .into_iter() + .map(|b| (b.feed_id, keywords(Some(b.words)).unwrap_or_default())) + .collect(); + Ok(feeds::Entity::find() + .all(&self.orm) + .await? + .into_iter() + .map(|f| { + let id = f.id.clone(); + let listing = FeedListing { + summary: FeedSummary { + entries: entries.get(&id).copied().unwrap_or(0), + downloaded: downloaded.get(&id).copied().unwrap_or(0), + title: f.title, + image: f.image, + last_checked: f.last_checked, + last_error: f.last_error, + orphaned: f.orphaned, + error_since: f.error_since, + category: f.category, + }, + ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64), + blocked: blocked.remove(&id).unwrap_or_default(), + unread: unread.get(&id).copied().unwrap_or(0), + }; + (id, listing) + }) + .collect()) + } +} + +/// One feed as the feed list shows it; see `Db::feed_list`. +#[derive(Debug, Default)] +pub struct FeedListing { + pub summary: FeedSummary, + pub ttl_mins: Option, + pub blocked: Vec, + pub unread: i64, } @@ -1808,6 +1879,33 @@ mod tests { assert!(!order_sql("x'; --", "asc").contains("x'")); } + #[tokio::test] + async fn the_feed_list_in_one_go_matches_asking_feed_by_feed() { + let db = Db::memory().await.unwrap(); + db.exec_for_test( + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false); + INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g'),(2,'f'); + INSERT INTO feeds (id, url, title, ttl_mins) VALUES ('f','u','F',30),('g','v','G',null); + INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0),('f','c',0),('g','d',0); + INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'f','a',true),(2,'f','b',true); + INSERT INTO hidden (user_id, feed_id, guid) VALUES (1,'f','c'); + INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES + (1,'f','a','u1','/tmp/a','done'),(2,'f','b','u2',null,'pending'); + INSERT INTO blocklists (user_id, feed_id, words) VALUES (1,'g','[\"spoiler\"]');", + ).await + .unwrap(); + let list = db.feed_list(1).await.unwrap(); + for id in ["f", "g"] { + let one = &list[id]; + let s = db.feed_summary(id).await.unwrap(); + assert_eq!((one.summary.entries, one.summary.downloaded), (s.entries, s.downloaded), "{id}"); + assert_eq!(one.unread, db.unread_count(1, id).await.unwrap(), "{id}"); + assert_eq!(one.blocked, db.blocklist(1, id).await.unwrap(), "{id}"); + assert_eq!(one.ttl_mins, db.http_state(id).await.unwrap().ttl_mins, "{id}"); + } + assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's + } + #[tokio::test] async fn only_artwork_a_feed_names_is_fetched_for_the_page() { let db = Db::memory().await.unwrap(); diff --git a/src/web.rs b/src/web.rs index 08478ae..ce7e3e2 100644 --- a/src/web.rs +++ b/src/web.rs @@ -744,6 +744,7 @@ async fn feeds( .collect(); let counts = state.ctx.db.subscriber_counts().await?; let pinned = state.ctx.db.pinned_feeds(user.id).await?; + let mut listed = state.ctx.db.feed_list(user.id).await?; let mut out = Vec::with_capacity(mine.len()); for sub in &subs { let (id, feed) = (&sub.id, &sub.cfg); @@ -751,8 +752,8 @@ async fn feeds( // the same fallback the scanner uses (`Db::subscribers`). let up = feed.group.as_deref().and_then(|g| mine.get(g)); let Some(mine) = mine.get(id) else { continue }; - let s = state.ctx.db.feed_summary(id).await?; - let st = state.ctx.db.http_state(id).await?; + // A feed not scanned yet has no row: blank, as feed_summary gave it. + let crate::db::FeedListing { summary: s, ttl_mins, blocked, unread } = listed.remove(id).unwrap_or_default(); out.push(FeedRow { id: id.clone(), url: feed.url.clone(), @@ -766,7 +767,7 @@ async fn feeds( .clone() .or_else(|| up.and_then(|u| u.keywords.clone())) .unwrap_or_else(|| feed.keywords.clone()), - blocked: state.ctx.db.blocklist(user.id, id).await?, + blocked, allow_explicit: mine .allow_explicit .or(up.and_then(|u| u.allow_explicit)) @@ -789,11 +790,11 @@ async fn feeds( .schedule .as_deref() .and_then(crate::config::parse_interval), - every_mins: crate::due_after(&cfg, feed, st.ttl_mins) / 60, + every_mins: crate::due_after(&cfg, feed, ttl_mins) / 60, last_checked: s.last_checked, next_check: s .last_checked - .map(|t| t + crate::due_after(&cfg, feed, st.ttl_mins) as i64), + .map(|t| t + crate::due_after(&cfg, feed, ttl_mins) as i64), failing: s .error_since .filter(|since| crate::db::now() - since >= FLAG_AFTER_SECS) @@ -803,7 +804,7 @@ async fn feeds( last_error: s.last_error, entries: s.entries, downloaded: s.downloaded, - unread: state.ctx.db.unread_count(user.id, id).await?, + unread, subscribers: counts.get(id).copied().unwrap_or(0), pinned: pinned.contains(id), });