Fetch feeds six ahead in a scan; reload the list only after a scan that checked something (#103, #104)

Fetching was 65-90% of a scan, each feed waiting for the one before: 13 s of fetches in a 20 s
refresh of 32 feeds. The scan now works out which feeds are due, fetches their bodies up to six
ahead in tasks of their own, and handles each in order as before, so database writes,
downloads and OPML syncs stay one at a time. A Patreon creator still fetches in scan_one.

The page reloaded /api/feeds, and /api/settings with it, on every scan_done: the scheduler
scans every minute, so each open page reloaded the list once a minute, 169 times an hour. It
now reloads only when the scan checked a feed, and asks for settings once, on first load.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- A scan fetches several feeds at once, so a refresh no longer waits on every site in turn.
+- An open page reloads the feed list only when a scan has checked something, not every minute.
 - Adding an address checks it first: a feed is added, a web page adds the feed it links, and anything else is refused with the reason, instead of being added and failing on every check.
 - A feed that was failing when its last subscriber left is forgotten, rather than kept with its error for good. One that worked, or has files on disk, is kept as before.
 - A feed that keeps failing is checked less and less often, waiting as long as it has been failing, up to once a day; it goes back to its schedule as soon as it works. Refreshing it still checks it at once.

src/main.rs

@@ -970,8 +970,9 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
     let in_scope = |s: &Sub| {
         scope.is_empty() || scope.contains(&s.id) || s.cfg.group.as_ref().is_some_and(|g| scope.contains(g))
     };
+    let mut due = vec![];
     for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id) && in_scope(s)) {
-        let (id, feed_cfg) = (&sub.id, &sub.cfg);
+        let id = &sub.id;
         let mut state = ctx.db.http_state(id).await?;
         // A scan someone asked for reads the feed in full. With the validators it only skipped the
         // wait: a feed that had not changed answered 304 and nothing was read (#77).
@@ -981,19 +982,47 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
         }
 
         if !force && let Some(last) = state.last_checked {
-            let due = last + due_after(&cfg, feed_cfg, state.ttl_mins, state.error_since, last) as i64;
-            if due > db::now() {
+            let at = last + due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last) as i64;
+            if at > db::now() {
                 ctx.out.emit(Event::FeedSkip {
                     feed: id.clone(),
-                    reason: format!("not due for {}", duration((due - db::now()) as u64)),
+                    reason: format!("not due for {}", duration((at - db::now()) as u64)),
                 });
                 continue;
             }
         }
+        due.push((sub, state));
+    }
 
+    // Each feed's body is fetched a few feeds ahead of its turn, in tasks of their own, and the
+    // feeds are then handled one at a time, in order, as before: database writes, downloads and
+    // OPML syncs stay one at a time. Fetched one after another, a scan waited on every site in
+    // turn, 65-90% of its time (#104). A Patreon creator fetches inside scan_one, its own way.
+    const AHEAD: usize = 6;
+    let mut bodies = futures_util::StreamExt::buffered(
+        futures_util::stream::iter(due.iter().map(|(sub, state)| {
+            let (client, feed_cfg) = (ctx.client.clone(), sub.cfg.clone());
+            let (etag, modified) = (state.etag.clone(), state.last_modified.clone());
+            let early = !feed::is_patreon_creator(&feed_cfg.url);
+            tokio::spawn(tracing::Instrument::instrument(
+                async move {
+                    if !early {
+                        return None;
+                    }
+                    Some(feed::fetch(&client, &feed_cfg, etag.as_deref(), modified.as_deref()).await)
+                },
+                tracing::Span::current(),
+            ))
+        })),
+        AHEAD,
+    );
+    for (sub, state) in &due {
+        let (id, feed_cfg) = (&sub.id, &sub.cfg);
+        // A task that panicked has no body; scan_one then fetches it itself.
+        let fetched = futures_util::StreamExt::next(&mut bodies).await.and_then(|j| j.ok()).flatten();
         scanned += 1;
         ctx.out.emit(Event::FeedStart { feed: id.clone() });
-        match scan_one(ctx, id, feed_cfg, &state, force).await {
+        match scan_one(ctx, id, feed_cfg, state, force, fetched).await {
             Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
                 feed: id.clone(),
                 new: s.new_entries,
@@ -1039,7 +1068,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
             scanned += 1;
             ctx.out.emit(Event::FeedStart { feed: id.clone() });
             let state = ctx.db.http_state(id).await?;
-            match scan_one(ctx, id, feed_cfg, &state, force).await {
+            match scan_one(ctx, id, feed_cfg, &state, force, None).await {
                 Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
                     feed: id.clone(),
                     new: s.new_entries,
@@ -1221,6 +1250,8 @@ async fn scan_one(
     feed_cfg: &config::Feed,
     state: &db::HttpState,
     force: bool,
+    // The body the scan already fetched for it, if it did (#104).
+    prefetched: Option<Result<feed::Fetched>>,
 ) -> Result<Outcome> {
     // A Patreon creator with more than one show is a list of feeds, like an OPML.
     if feed::is_patreon_creator(&feed_cfg.url) {
@@ -1247,13 +1278,10 @@ async fn scan_one(
         }
     }
 
-    let mut fetched = feed::fetch(
-        &ctx.client,
-        feed_cfg,
-        state.etag.as_deref(),
-        state.last_modified.as_deref(),
-    )
-    .await?;
+    let mut fetched = match prefetched {
+        Some(got) => got?,
+        None => feed::fetch(&ctx.client, feed_cfg, state.etag.as_deref(), state.last_modified.as_deref()).await?,
+    };
 
     // A 304 while nothing is stored means the validator has outlived the data -- a restore
     // from backup, a manual edit, a cleanup that removed entries. Believe the database over

web/src/events.ts

@@ -40,7 +40,9 @@ function connect(){
     // No toast: a scan of every feed raised one per failure, to everyone. The feed list's
     // red ! marks the feed instead, and its page says why.
     else if(ev.ev==='feed_error'){ setScanning(ev.feed,false); if(mine(ev.feed)) refreshFeeds(); }
-    else if(ev.ev==='scan_done'){ scanning.clear(); paintScanning(); refreshFeeds(); refreshEntries(); }
+    // Only a scan that checked something: the scheduler scans every minute, due or not, and
+    // every open page reloaded the whole list each time (#103).
+    else if(ev.ev==='scan_done'){ scanning.clear(); paintScanning(); if(ev.feeds){ refreshFeeds(); refreshEntries(); } }
   };
   sse.onerror=()=>{ sse.close(); setTimeout(connect,4000); };
 }

web/src/feeds.ts

@@ -1,7 +1,9 @@
 /* ---------------- feeds ---------------- */
 async function loadFeeds(keepSel?: boolean){
   S.feeds = await api('/api/feeds');
-  api('/api/settings').then(g=>{globalMax=g.max_new_per_check}).catch(()=>{});
+  // Only for a hint in a feed's settings, so once, on the page's first load, not on every reload
+  // of the list (#103).
+  if(!keepSel) api('/api/settings').then(g=>{globalMax=g.max_new_per_check}).catch(()=>{});
   renderFeeds();
   // Land back where you were; a feed you no longer subscribe to, or a first visit, goes to
   // All Subscriptions rather than picking one alphabetically. Nothing to land on at all (a