The directory's public pages read the list once a minute (#341)

/, /all, /search and /category are anyone's to ask for (#215), and each
called Store::list: the whole catalogue, joined with the snapshots and
grouped, on the 10-connection pool the refresher and every server's
/v1 calls share, on the Postgres that holds production ipx's database.
The pages' Cache-Control did nothing for a request with a query string
of its own, such as /search?q=<random>.

Directory::pages_list keeps the list for a minute behind a mutex held
through the read, so a burst of misses waits on one query rather than
each sending its own. Filtering and rendering still happen per request,
in memory. /v1/feeds, behind a token, still reads the database.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/directory/CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Security
 
 - HTTPS connections no longer trust certificates from emSign's distrusted intermediates, and certificate checks use constant-time comparisons: rustls-webpki, webpki-roots and ctutils are updated (#351).
+- The public pages read the list from the database at most once a minute, however many people ask. Each page asked for the whole catalogue, so a flood of requests with query strings of their own reached the database every time (#341).
 
 ## [1.3.0] - 2026-10-19
 

crates/directory/src/lib.rs

@@ -127,8 +127,13 @@ pub struct Directory {
     client: reqwest::Client,
     /// Wakes the refresher, so a feed a snapshot brings in is fetched now, not in a minute.
     wake: tokio::sync::Notify,
+    /// The public pages' copy of the list, and when it was read.
+    pages_list: tokio::sync::Mutex<Option<(Instant, Arc<Vec<Listing>>)>>,
 }
 
+/// How long the public pages show the same copy of the list.
+const PAGES_LIST_FOR: Duration = Duration::from_secs(60);
+
 pub type Shared = Arc<Directory>;
 
 impl Directory {
@@ -140,7 +145,29 @@ impl Directory {
             .redirect(reqwest::redirect::Policy::none())
             .user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION")))
             .build()?;
-        Ok(Arc::new(Self { store, tokens: RwLock::new(tokens), client, wake: tokio::sync::Notify::new() }))
+        Ok(Arc::new(Self {
+            store,
+            tokens: RwLock::new(tokens),
+            client,
+            wake: tokio::sync::Notify::new(),
+            pages_list: Default::default(),
+        }))
+    }
+
+    /// The whole list for the public pages, read from the database at most once a minute. Anyone
+    /// may ask for a page, and each asked the database for the whole catalogue on the pool the
+    /// refresher and every server's /v1 calls share; a query string of its own got past any
+    /// cache in front (#341). The lock is held through the read, so a burst waits on one query.
+    pub async fn pages_list(&self) -> anyhow::Result<Arc<Vec<Listing>>> {
+        let mut held = self.pages_list.lock().await;
+        if let Some((at, list)) = held.as_ref()
+            && at.elapsed() < PAGES_LIST_FOR
+        {
+            return Ok(list.clone());
+        }
+        let list = Arc::new(self.store.list().await?);
+        *held = Some((Instant::now(), list.clone()));
+        Ok(list)
     }
 
     /// Takes the tokens of a config file read again, and drops the snapshots of any that went.
@@ -541,6 +568,16 @@ mod tests {
         assert_eq!(got["accepted"], 1);
     }
 
+    #[tokio::test]
+    async fn the_public_pages_read_the_list_once_a_minute() {
+        let (_, dir) = app().await;
+        let first = dir.pages_list().await.unwrap();
+        assert!(Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()), "a second page asks the database nothing");
+        // Once the copy is a minute old, the next page reads the list again.
+        dir.pages_list.lock().await.as_mut().unwrap().0 -= PAGES_LIST_FOR;
+        assert!(!Arc::ptr_eq(&first, &dir.pages_list().await.unwrap()));
+    }
+
     #[tokio::test]
     async fn answers_come_compressed_as_the_caller_asks() {
         let (app, _) = app().await;

crates/directory/src/site.rs

@@ -58,17 +58,17 @@ impl Filter {
 }
 
 async fn home(State(dir): State<Shared>) -> Response {
-    let all = match dir.store.list().await {
+    let all = match dir.pages_list().await {
         Ok(a) => a,
         Err(e) => return failed(e),
     };
-    let mut top = all.clone();
+    let mut top = (*all).clone();
     top.sort_by(|a, b| {
         b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase()))
     });
     top.truncate(10);
     let mut cats: BTreeMap<&str, Vec<&Listing>> = BTreeMap::new();
-    for l in &all {
+    for l in all.iter() {
         cats.entry(l.category.as_deref().unwrap_or("Other")).or_default().push(l);
     }
     let mut b = String::new();
@@ -123,8 +123,8 @@ async fn category(State(dir): State<Shared>, Path(c): Path<String>, Query(f): Qu
 
 /// A page of tiles: those `pick` takes, of the kind and subcategory asked for, A to Z.
 async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn(&Listing) -> bool) -> Response {
-    let mut ls: Vec<Listing> = match dir.store.list().await {
+    let mut ls: Vec<Listing> = match dir.pages_list().await {
-        Ok(a) => a.into_iter().filter(|l| pick(l)).collect(),
+        Ok(a) => a.iter().filter(|l| pick(l)).cloned().collect(),
         Err(e) => return failed(e),
     };
     if ls.is_empty() && here.starts_with("/category/") {