Count as queued only what a scan will download on its own (#113)

Every file in state 'pending' counted as waiting to download: 4420 in production, across 12
shows. Since #97 a scan only downloads among a feed's newest max_new_per_check items, so those
were back-catalogue episodes no scan would take; the real queue was 0.

A new state, 'held': listed and downloadable by hand, but outside the feed's newest items, or of
a feed nothing downloads automatically. Db::hold_back moves a feed's waiting files between
'pending' and 'held' each time the feed is due, changed or not, and again after its items are
stored, so a new episode, a raised limit or auto-download turned on or off moves them. A held
file keeps its item's place among the newest, as a downloaded one does. 'pending' now means
queued, so ipx status, /api/status and the dashboard read true without changing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- The download queue, in `ipx status`, `/api/status` and the dashboard, counts only what iPX will download on its own: older episodes beyond a show's limit, and files of feeds that do not download automatically, are listed but no longer counted as waiting.
 - Artwork published on http is stored on https when its host serves it there, so the page loads it directly; the rest still comes through iPX.
 - An iPhone adding the site to its home screen finds the icon at the first address it tries.
 - A feed whose server hangs no longer holds up every scan: a feed gets 30 seconds, and connecting anywhere 10.

src/db.rs

@@ -652,6 +652,35 @@ impl Db {
     /// through every show's history: 4420 files queued across 12 shows on the default of 3
     /// (#97). Unlimited, 0 in the settings, is the whole back catalogue, for an archive. An item
     /// whose files were all skipped by a filter does not hold one of the places.
+    /// Puts a feed's waiting files in the queue or out of it: 'pending' for those among its
+    /// `limit` newest items, which the scan downloads on its own, and 'held' for the rest, listed
+    /// to download by hand. With all of them 'pending', the queue counted 4420 back-catalogue
+    /// files no scan would ever take (#97, #113). 0 holds them all, for a feed nobody downloads
+    /// automatically. A held file still holds its item's place among the newest.
+    #[tracing::instrument(skip_all)]
+    pub async fn hold_back(&self, feed_id: &str, limit: usize) -> Result<()> {
+        let newest = "SELECT n.guid FROM entries n
+                       WHERE n.feed_id = $1
+                         AND EXISTS (SELECT 1 FROM enclosures y
+                                      WHERE y.feed_id = n.feed_id AND y.guid = n.guid
+                                        AND y.state <> 'skipped')
+                       ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC
+                       LIMIT $2";
+        let limit: sea_orm::Value = (limit.min(i64::MAX as usize) as i64).into();
+        self.exec(
+            &format!("UPDATE enclosures SET state = 'held' WHERE feed_id = $1 AND state = 'pending' AND guid NOT IN ({newest})"),
+            vec![feed_id.into(), limit.clone()],
+        )
+        .await?;
+        // And back, when a limit is raised or a feed downloads again.
+        self.exec(
+            &format!("UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"),
+            vec![feed_id.into(), limit],
+        )
+        .await?;
+        Ok(())
+    }
+
     #[tracing::instrument(skip_all)]
     pub async fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
         self.rows(
@@ -2130,6 +2159,33 @@ mod tests {
         assert!(db.prune_abandoned_failures().await.unwrap().is_empty());
     }
 
+    #[tokio::test]
+    async fn only_the_newest_files_stay_queued_and_the_rest_are_held() {
+        let db = Db::memory().await.unwrap();
+        db.exec_for_test(
+            "INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
+                ('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400);
+             INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
+                (1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
+                (3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);",
+        ).await
+        .unwrap();
+        let states = || async {
+            let mut v: Vec<(i64, String)> = db.rows("SELECT id, state FROM enclosures ORDER BY id", vec![]).await.unwrap()
+                .iter().map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap())).collect();
+            v.sort();
+            v.into_iter().map(|(_, s)| s).collect::<Vec<_>>()
+        };
+        db.hold_back("f", 2).await.unwrap();              // newest two: e4, e3
+        assert_eq!(states().await, ["held", "held", "done", "pending"]);
+        db.hold_back("f", 3).await.unwrap();              // a limit raised brings e2 back
+        assert_eq!(states().await, ["held", "pending", "done", "pending"]);
+        db.hold_back("f", 0).await.unwrap();              // a feed nobody downloads: all held
+        assert_eq!(states().await, ["held", "held", "done", "held"]);
+        db.hold_back("f", usize::MAX).await.unwrap();     // unlimited, an archive: all queued
+        assert_eq!(states().await, ["pending", "pending", "done", "pending"]);
+    }
+
     #[tokio::test]
     async fn a_limit_takes_the_newest_episodes_not_the_back_catalogue() {
         let db = Db::memory().await.unwrap();

src/entity.rs

@@ -102,6 +102,10 @@ pub mod enclosures {
         pub length: Option<i64>,
         #[sea_orm(column_type = "Text", nullable)]
         pub path: Option<String>,
+        /// 'pending' is queued: a scan downloads it on its own. 'held' is listed but outside the
+        /// feed's newest max_new_per_check, or its feed downloads nothing automatically; it can
+        /// be downloaded by hand (`Db::hold_back`). 'skipped' is filtered out ('last_error' says
+        /// why), then 'downloading', 'done', 'error', 'reaped'.
         #[sea_orm(column_type = "Text")]
         pub state: String,
         #[sea_orm(default_value = 0)]

src/ipc.rs

@@ -69,7 +69,7 @@ impl Event {
                 *bytes as f64 / 1_048_576.0
             ),
             Event::Status { feeds, pending, downloaded } => {
-                format!("{feeds} feeds, {pending} pending, {downloaded} downloaded")
+                format!("{feeds} feeds, {pending} queued to download, {downloaded} downloaded")
             }
             Event::Error { msg } => format!("error: {msg}"),
             // Noise in a terminal; a UI still gets them on the socket.

src/main.rs

@@ -1333,6 +1333,10 @@ async fn scan_one(
     // The body the scan already fetched for it, if it did (#104).
     prefetched: Option<Result<feed::Fetched>>,
 ) -> Result<Outcome> {
+    // The queue follows the feed's settings each time it is due, changed or not, so files a
+    // limit or auto-download no longer reaches stop counting as waiting (#113).
+    let policy = policy_for(ctx, id, feed_cfg).await?;
+    ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?;
     // A Patreon creator with more than one show is a list of feeds, like an OPML.
     if feed::is_patreon_creator(&feed_cfg.url) {
         match feed::patreon_shows(&ctx.client, &feed_cfg.url).await {
@@ -1456,7 +1460,6 @@ async fn scan_one(
         parsed.category.as_deref(),
     ).await?;
 
-    let policy = policy_for(ctx, id, feed_cfg).await?;
     if let Some(parent) = &feed_cfg.group {
         let listed: Vec<(&str, &str)> = parsed
             .entries
@@ -1503,6 +1506,9 @@ async fn scan_one(
     if scan.new_entries > 0 {
         ctx.db.rehide(id).await?;   // what is new may hold someone's blocked words
     }
+    // A new item takes a place among the newest and the oldest of them leaves the queue; a file a
+    // filter lets through again may be outside them.
+    ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?;
 
     let budget = policy.budget;
     if policy.auto_download && budget > 0 {

web/src/items.ts

@@ -328,7 +328,7 @@ function encBox(x){
   // Nothing on disk. For an image or a PDF you usually just want to look at it, so link
   // straight to the publisher's copy in a new tab -- no download, and nothing proxied
   // through here, which would make ipx a fetch-anything relay.
-  const viewable = !isPlayable(x) && x.state !== 'pending';
+  const viewable = !isPlayable(x) && x.state !== 'pending' && x.state !== 'held';
   return `<div class="encbox">
     ${kindIcon(x)}
     <span class="meta" style="flex:1">${size}</span>