From a387012c693b8ee73d8e8d7ab7838a3e22d14923 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 2 Oct 2026 19:47:52 +0000 Subject: [PATCH] Count as queued only what a scan will download on its own (#113) Every file in state 'pending' counted as waiting to download: 4420 in production, across 12 shows. Since #97 a scan only downloads among a feed's newest max_new_per_check items, so those were back-catalogue episodes no scan would take; the real queue was 0. A new state, 'held': listed and downloadable by hand, but outside the feed's newest items, or of a feed nothing downloads automatically. Db::hold_back moves a feed's waiting files between 'pending' and 'held' each time the feed is due, changed or not, and again after its items are stored, so a new episode, a raised limit or auto-download turned on or off moves them. A held file keeps its item's place among the newest, as a downloaded one does. 'pending' now means queued, so ipx status, /api/status and the dashboard read true without changing. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + src/db.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++++ src/entity.rs | 4 ++++ src/ipc.rs | 2 +- src/main.rs | 8 ++++++- web/src/items.ts | 2 +- 6 files changed, 70 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2a184f..6200acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The download queue, in `ipx status`, `/api/status` and the dashboard, counts only what iPX will download on its own: older episodes beyond a show's limit, and files of feeds that do not download automatically, are listed but no longer counted as waiting. - Artwork published on http is stored on https when its host serves it there, so the page loads it directly; the rest still comes through iPX. - An iPhone adding the site to its home screen finds the icon at the first address it tries. - A feed whose server hangs no longer holds up every scan: a feed gets 30 seconds, and connecting anywhere 10. diff --git a/src/db.rs b/src/db.rs index efcdf36..1f9ebce 100644 --- a/src/db.rs +++ b/src/db.rs @@ -652,6 +652,35 @@ impl Db { /// through every show's history: 4420 files queued across 12 shows on the default of 3 /// (#97). Unlimited, 0 in the settings, is the whole back catalogue, for an archive. An item /// whose files were all skipped by a filter does not hold one of the places. + /// Puts a feed's waiting files in the queue or out of it: 'pending' for those among its + /// `limit` newest items, which the scan downloads on its own, and 'held' for the rest, listed + /// to download by hand. With all of them 'pending', the queue counted 4420 back-catalogue + /// files no scan would ever take (#97, #113). 0 holds them all, for a feed nobody downloads + /// automatically. A held file still holds its item's place among the newest. + #[tracing::instrument(skip_all)] + pub async fn hold_back(&self, feed_id: &str, limit: usize) -> Result<()> { + let newest = "SELECT n.guid FROM entries n + WHERE n.feed_id = $1 + AND EXISTS (SELECT 1 FROM enclosures y + WHERE y.feed_id = n.feed_id AND y.guid = n.guid + AND y.state <> 'skipped') + ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC + LIMIT $2"; + let limit: sea_orm::Value = (limit.min(i64::MAX as usize) as i64).into(); + self.exec( + &format!("UPDATE enclosures SET state = 'held' WHERE feed_id = $1 AND state = 'pending' AND guid NOT IN ({newest})"), + vec![feed_id.into(), limit.clone()], + ) + .await?; + // And back, when a limit is raised or a feed downloads again. + self.exec( + &format!("UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"), + vec![feed_id.into(), limit], + ) + .await?; + Ok(()) + } + #[tracing::instrument(skip_all)] pub async fn pending(&self, feed_id: &str, limit: usize) -> Result> { self.rows( @@ -2130,6 +2159,33 @@ mod tests { assert!(db.prune_abandoned_failures().await.unwrap().is_empty()); } + #[tokio::test] + async fn only_the_newest_files_stay_queued_and_the_rest_are_held() { + let db = Db::memory().await.unwrap(); + db.exec_for_test( + "INSERT INTO entries (feed_id, guid, first_seen, published) VALUES + ('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400); + INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES + (1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null), + (3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);", + ).await + .unwrap(); + let states = || async { + let mut v: Vec<(i64, String)> = db.rows("SELECT id, state FROM enclosures ORDER BY id", vec![]).await.unwrap() + .iter().map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap())).collect(); + v.sort(); + v.into_iter().map(|(_, s)| s).collect::>() + }; + db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3 + assert_eq!(states().await, ["held", "held", "done", "pending"]); + db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back + assert_eq!(states().await, ["held", "pending", "done", "pending"]); + db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held + assert_eq!(states().await, ["held", "held", "done", "held"]); + db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued + assert_eq!(states().await, ["pending", "pending", "done", "pending"]); + } + #[tokio::test] async fn a_limit_takes_the_newest_episodes_not_the_back_catalogue() { let db = Db::memory().await.unwrap(); diff --git a/src/entity.rs b/src/entity.rs index 8906e18..6ecfdf7 100644 --- a/src/entity.rs +++ b/src/entity.rs @@ -102,6 +102,10 @@ pub mod enclosures { pub length: Option, #[sea_orm(column_type = "Text", nullable)] pub path: Option, + /// 'pending' is queued: a scan downloads it on its own. 'held' is listed but outside the + /// feed's newest max_new_per_check, or its feed downloads nothing automatically; it can + /// be downloaded by hand (`Db::hold_back`). 'skipped' is filtered out ('last_error' says + /// why), then 'downloading', 'done', 'error', 'reaped'. #[sea_orm(column_type = "Text")] pub state: String, #[sea_orm(default_value = 0)] diff --git a/src/ipc.rs b/src/ipc.rs index 40c048f..660edff 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -69,7 +69,7 @@ impl Event { *bytes as f64 / 1_048_576.0 ), Event::Status { feeds, pending, downloaded } => { - format!("{feeds} feeds, {pending} pending, {downloaded} downloaded") + format!("{feeds} feeds, {pending} queued to download, {downloaded} downloaded") } Event::Error { msg } => format!("error: {msg}"), // Noise in a terminal; a UI still gets them on the socket. diff --git a/src/main.rs b/src/main.rs index d34268b..a5de343 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1333,6 +1333,10 @@ async fn scan_one( // The body the scan already fetched for it, if it did (#104). prefetched: Option>, ) -> Result { + // The queue follows the feed's settings each time it is due, changed or not, so files a + // limit or auto-download no longer reaches stop counting as waiting (#113). + let policy = policy_for(ctx, id, feed_cfg).await?; + ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?; // A Patreon creator with more than one show is a list of feeds, like an OPML. if feed::is_patreon_creator(&feed_cfg.url) { match feed::patreon_shows(&ctx.client, &feed_cfg.url).await { @@ -1456,7 +1460,6 @@ async fn scan_one( parsed.category.as_deref(), ).await?; - let policy = policy_for(ctx, id, feed_cfg).await?; if let Some(parent) = &feed_cfg.group { let listed: Vec<(&str, &str)> = parsed .entries @@ -1503,6 +1506,9 @@ async fn scan_one( if scan.new_entries > 0 { ctx.db.rehide(id).await?; // what is new may hold someone's blocked words } + // A new item takes a place among the newest and the oldest of them leaves the queue; a file a + // filter lets through again may be outside them. + ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?; let budget = policy.budget; if policy.auto_download && budget > 0 { diff --git a/web/src/items.ts b/web/src/items.ts index d8dad5c..f289c20 100644 --- a/web/src/items.ts +++ b/web/src/items.ts @@ -328,7 +328,7 @@ function encBox(x){ // Nothing on disk. For an image or a PDF you usually just want to look at it, so link // straight to the publisher's copy in a new tab -- no download, and nothing proxied // through here, which would make ipx a fetch-anything relay. - const viewable = !isPlayable(x) && x.state !== 'pending'; + const viewable = !isPlayable(x) && x.state !== 'pending' && x.state !== 'held'; return `
${kindIcon(x)} ${size}