From fc04a741a65404d1da92fbf18fbe17a3208ff096 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 23 Oct 2026 14:06:19 +0000 Subject: [PATCH] The directory's public pages read the list once a minute (#341) /, /all, /search and /category are anyone's to ask for (#215), and each called Store::list: the whole catalogue, joined with the snapshots and grouped, on the 10-connection pool the refresher and every server's /v1 calls share, on the Postgres that holds production ipx's database. The pages' Cache-Control did nothing for a request with a query string of its own, such as /search?q=. Directory::pages_list keeps the list for a minute behind a mutex held through the read, so a burst of misses waits on one query rather than each sending its own. Filtering and rendering still happen per request, in memory. /v1/feeds, behind a token, still reads the database. Co-Authored-By: Claude Opus 5.5 --- crates/directory/CHANGELOG.md | 1 + crates/directory/src/lib.rs | 39 ++++++++++++++++++++++++++++++++++- crates/directory/src/site.rs | 10 ++++----- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index 533da9d..77d8540 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - HTTPS connections no longer trust certificates from emSign's distrusted intermediates, and certificate checks use constant-time comparisons: rustls-webpki, webpki-roots and ctutils are updated (#351). +- The public pages read the list from the database at most once a minute, however many people ask. Each page asked for the whole catalogue, so a flood of requests with query strings of their own reached the database every time (#341). ## [1.3.0] - 2026-10-19 diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 3599c0b..424534a 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -127,8 +127,13 @@ pub struct Directory { client: reqwest::Client, /// Wakes the refresher, so a feed a snapshot brings in is fetched now, not in a minute. wake: tokio::sync::Notify, + /// The public pages' copy of the list, and when it was read. + pages_list: tokio::sync::Mutex>)>>, } +/// How long the public pages show the same copy of the list. +const PAGES_LIST_FOR: Duration = Duration::from_secs(60); + pub type Shared = Arc; impl Directory { @@ -140,7 +145,29 @@ impl Directory { .redirect(reqwest::redirect::Policy::none()) .user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION"))) .build()?; - Ok(Arc::new(Self { store, tokens: RwLock::new(tokens), client, wake: tokio::sync::Notify::new() })) + Ok(Arc::new(Self { + store, + tokens: RwLock::new(tokens), + client, + wake: tokio::sync::Notify::new(), + pages_list: Default::default(), + })) + } + + /// The whole list for the public pages, read from the database at most once a minute. Anyone + /// may ask for a page, and each asked the database for the whole catalogue on the pool the + /// refresher and every server's /v1 calls share; a query string of its own got past any + /// cache in front (#341). The lock is held through the read, so a burst waits on one query. + pub async fn pages_list(&self) -> anyhow::Result>> { + let mut held = self.pages_list.lock().await; + if let Some((at, list)) = held.as_ref() + && at.elapsed() < PAGES_LIST_FOR + { + return Ok(list.clone()); + } + let list = Arc::new(self.store.list().await?); + *held = Some((Instant::now(), list.clone())); + Ok(list) } /// Takes the tokens of a config file read again, and drops the snapshots of any that went. @@ -541,6 +568,16 @@ mod tests { assert_eq!(got["accepted"], 1); } + #[tokio::test] + async fn the_public_pages_read_the_list_once_a_minute() { + let (_, dir) = app().await; + let first = dir.pages_list().await.unwrap(); + assert!(Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()), "a second page asks the database nothing"); + // Once the copy is a minute old, the next page reads the list again. + dir.pages_list.lock().await.as_mut().unwrap().0 -= PAGES_LIST_FOR; + assert!(!Arc::ptr_eq(&first, &dir.pages_list().await.unwrap())); + } + #[tokio::test] async fn answers_come_compressed_as_the_caller_asks() { let (app, _) = app().await; diff --git a/crates/directory/src/site.rs b/crates/directory/src/site.rs index 9b0926d..f0c072f 100644 --- a/crates/directory/src/site.rs +++ b/crates/directory/src/site.rs @@ -58,17 +58,17 @@ impl Filter { } async fn home(State(dir): State) -> Response { - let all = match dir.store.list().await { + let all = match dir.pages_list().await { Ok(a) => a, Err(e) => return failed(e), }; - let mut top = all.clone(); + let mut top = (*all).clone(); top.sort_by(|a, b| { b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase())) }); top.truncate(10); let mut cats: BTreeMap<&str, Vec<&Listing>> = BTreeMap::new(); - for l in &all { + for l in all.iter() { cats.entry(l.category.as_deref().unwrap_or("Other")).or_default().push(l); } let mut b = String::new(); @@ -123,8 +123,8 @@ async fn category(State(dir): State, Path(c): Path, Query(f): Qu /// A page of tiles: those `pick` takes, of the kind and subcategory asked for, A to Z. async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn(&Listing) -> bool) -> Response { - let mut ls: Vec = match dir.store.list().await { - Ok(a) => a.into_iter().filter(|l| pick(l)).collect(), + let mut ls: Vec = match dir.pages_list().await { + Ok(a) => a.iter().filter(|l| pick(l)).cloned().collect(), Err(e) => return failed(e), }; if ls.is_empty() && here.starts_with("/category/") {