Load the feed list in five queries, not six per feed (#94)
GET /api/feeds called feed_summary, http_state, blocklist and unread_count for every feed: about 950 round trips to Postgres for 160 feeds, 320 ms on every page load. Db::feed_list asks for the feed rows, entry counts, download counts, the person's unread counts and block lists once each, and the handler reads from that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
98
src/db.rs
98
src/db.rs
@@ -333,6 +333,77 @@ impl Db {
|
||||
sum.downloaded = self.downloaded_count(feed_id).await?;
|
||||
Ok(sum)
|
||||
}
|
||||
|
||||
/// What the feed list shows of every feed, for one person, in five queries whatever the
|
||||
/// number of feeds. Asked feed by feed (feed_summary, http_state, blocklist, unread_count)
|
||||
/// it was six round trips a feed, about 950 for 160 feeds and 320 ms a page load (#94).
|
||||
pub async fn feed_list(&self, user_id: i64) -> Result<std::collections::HashMap<String, FeedListing>> {
|
||||
use std::collections::HashMap;
|
||||
let counts = |sql: &'static str, args: Vec<sea_orm::Value>| async move {
|
||||
self.rows(sql, args)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| Ok((r.try_get::<String>("", "feed_id")?, r.try_get::<i64>("", "n")?)))
|
||||
.collect::<Result<HashMap<_, _>>>()
|
||||
};
|
||||
let entries = counts("SELECT feed_id, count(*) AS n FROM entries GROUP BY feed_id", vec![]).await?;
|
||||
let downloaded =
|
||||
counts("SELECT feed_id, count(*) AS n FROM enclosures WHERE path IS NOT NULL GROUP BY feed_id", vec![])
|
||||
.await?;
|
||||
let unread = counts(
|
||||
"SELECT e.feed_id, count(*) AS n FROM entries e
|
||||
JOIN subscriptions sub ON sub.user_id = $1 AND sub.feed_id = e.feed_id
|
||||
LEFT JOIN entry_state s
|
||||
ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid
|
||||
WHERE NOT coalesce(s.read, false)
|
||||
AND NOT EXISTS (SELECT 1 FROM hidden h
|
||||
WHERE h.user_id = $1 AND h.feed_id = e.feed_id AND h.guid = e.guid)
|
||||
GROUP BY e.feed_id",
|
||||
vec![user_id.into()],
|
||||
)
|
||||
.await?;
|
||||
let mut blocked: HashMap<String, Vec<String>> = blocklists::Entity::find()
|
||||
.filter(blocklists::Column::UserId.eq(user_id))
|
||||
.all(&self.orm)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|b| (b.feed_id, keywords(Some(b.words)).unwrap_or_default()))
|
||||
.collect();
|
||||
Ok(feeds::Entity::find()
|
||||
.all(&self.orm)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|f| {
|
||||
let id = f.id.clone();
|
||||
let listing = FeedListing {
|
||||
summary: FeedSummary {
|
||||
entries: entries.get(&id).copied().unwrap_or(0),
|
||||
downloaded: downloaded.get(&id).copied().unwrap_or(0),
|
||||
title: f.title,
|
||||
image: f.image,
|
||||
last_checked: f.last_checked,
|
||||
last_error: f.last_error,
|
||||
orphaned: f.orphaned,
|
||||
error_since: f.error_since,
|
||||
category: f.category,
|
||||
},
|
||||
ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64),
|
||||
blocked: blocked.remove(&id).unwrap_or_default(),
|
||||
unread: unread.get(&id).copied().unwrap_or(0),
|
||||
};
|
||||
(id, listing)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// One feed as the feed list shows it; see `Db::feed_list`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct FeedListing {
|
||||
pub summary: FeedSummary,
|
||||
pub ttl_mins: Option<u64>,
|
||||
pub blocked: Vec<String>,
|
||||
pub unread: i64,
|
||||
}
|
||||
|
||||
|
||||
@@ -1808,6 +1879,33 @@ mod tests {
|
||||
assert!(!order_sql("x'; --", "asc").contains("x'"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_feed_list_in_one_go_matches_asking_feed_by_feed() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
|
||||
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g'),(2,'f');
|
||||
INSERT INTO feeds (id, url, title, ttl_mins) VALUES ('f','u','F',30),('g','v','G',null);
|
||||
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0),('f','c',0),('g','d',0);
|
||||
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'f','a',true),(2,'f','b',true);
|
||||
INSERT INTO hidden (user_id, feed_id, guid) VALUES (1,'f','c');
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
|
||||
(1,'f','a','u1','/tmp/a','done'),(2,'f','b','u2',null,'pending');
|
||||
INSERT INTO blocklists (user_id, feed_id, words) VALUES (1,'g','[\"spoiler\"]');",
|
||||
).await
|
||||
.unwrap();
|
||||
let list = db.feed_list(1).await.unwrap();
|
||||
for id in ["f", "g"] {
|
||||
let one = &list[id];
|
||||
let s = db.feed_summary(id).await.unwrap();
|
||||
assert_eq!((one.summary.entries, one.summary.downloaded), (s.entries, s.downloaded), "{id}");
|
||||
assert_eq!(one.unread, db.unread_count(1, id).await.unwrap(), "{id}");
|
||||
assert_eq!(one.blocked, db.blocklist(1, id).await.unwrap(), "{id}");
|
||||
assert_eq!(one.ttl_mins, db.http_state(id).await.unwrap().ttl_mins, "{id}");
|
||||
}
|
||||
assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_artwork_a_feed_names_is_fetched_for_the_page() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user