rays/ipx
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Count as queued only what a scan will download on its own (#113)

Browse Source 
Every file in state 'pending' counted as waiting to download: 4420 in production, across 12
shows. Since #97 a scan only downloads among a feed's newest max_new_per_check items, so those
were back-catalogue episodes no scan would take; the real queue was 0.

A new state, 'held': listed and downloadable by hand, but outside the feed's newest items, or of
a feed nothing downloads automatically. Db::hold_back moves a feed's waiting files between
'pending' and 'held' each time the feed is due, changed or not, and again after its items are
stored, so a new episode, a raised limit or auto-download turned on or off moves them. A held
file keeps its item's place among the newest, as a downloaded one does. 'pending' now means
queued, so ipx status, /api/status and the dashboard read true without changing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-02 19:47:52 +00:00
parent a00516687a
commit a387012c69
6 changed files with 70 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- The download queue, in `ipx status`, `/api/status` and the dashboard, counts only what iPX will download on its own: older episodes beyond a show's limit, and files of feeds that do not download automatically, are listed but no longer counted as waiting.
- Artwork published on http is stored on https when its host serves it there, so the page loads it directly; the rest still comes through iPX.
- An iPhone adding the site to its home screen finds the icon at the first address it tries.
- A feed whose server hangs no longer holds up every scan: a feed gets 30 seconds, and connecting anywhere 10.

56
src/db.rs
View File

@@ -652,6 +652,35 @@ impl Db {
/// through every show's history: 4420 files queued across 12 shows on the default of 3
/// (#97). Unlimited, 0 in the settings, is the whole back catalogue, for an archive. An item
/// whose files were all skipped by a filter does not hold one of the places.
/// Puts a feed's waiting files in the queue or out of it: 'pending' for those among its
/// `limit` newest items, which the scan downloads on its own, and 'held' for the rest, listed
/// to download by hand. With all of them 'pending', the queue counted 4420 back-catalogue
/// files no scan would ever take (#97, #113). 0 holds them all, for a feed nobody downloads
/// automatically. A held file still holds its item's place among the newest.
#[tracing::instrument(skip_all)]
pub async fn hold_back(&self, feed_id: &str, limit: usize) -> Result<()> {
let newest = "SELECT n.guid FROM entries n
WHERE n.feed_id = $1
AND EXISTS (SELECT 1 FROM enclosures y
WHERE y.feed_id = n.feed_id AND y.guid = n.guid
AND y.state <> 'skipped')
ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC
LIMIT $2";
let limit: sea_orm::Value = (limit.min(i64::MAX as usize) as i64).into();
self.exec(
&format!("UPDATE enclosures SET state = 'held' WHERE feed_id = $1 AND state = 'pending' AND guid NOT IN ({newest})"),
vec![feed_id.into(), limit.clone()],
)
.await?;
// And back, when a limit is raised or a feed downloads again.
self.exec(
&format!("UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"),
vec![feed_id.into(), limit],
)
.await?;
Ok(())
}
#[tracing::instrument(skip_all)]
pub async fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
self.rows(
@@ -2130,6 +2159,33 @@ mod tests {
assert!(db.prune_abandoned_failures().await.unwrap().is_empty());
}
#[tokio::test]
async fn only_the_newest_files_stay_queued_and_the_rest_are_held() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400);
INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);",
).await
.unwrap();
let states = || async {
let mut v: Vec<(i64, String)> = db.rows("SELECT id, state FROM enclosures ORDER BY id", vec![]).await.unwrap()
.iter().map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap())).collect();
v.sort();
v.into_iter().map(|(_, s)| s).collect::<Vec<_>>()
};
db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3
assert_eq!(states().await, ["held", "held", "done", "pending"]);
db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back
assert_eq!(states().await, ["held", "pending", "done", "pending"]);
db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held
assert_eq!(states().await, ["held", "held", "done", "held"]);
db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued
assert_eq!(states().await, ["pending", "pending", "done", "pending"]);
}
#[tokio::test]
async fn a_limit_takes_the_newest_episodes_not_the_back_catalogue() {
let db = Db::memory().await.unwrap();

4
src/entity.rs
View File

@@ -102,6 +102,10 @@ pub mod enclosures {
pub length: Option<i64>,
#[sea_orm(column_type = "Text", nullable)]
pub path: Option<String>,
/// 'pending' is queued: a scan downloads it on its own. 'held' is listed but outside the
/// feed's newest max_new_per_check, or its feed downloads nothing automatically; it can
/// be downloaded by hand (`Db::hold_back`). 'skipped' is filtered out ('last_error' says
/// why), then 'downloading', 'done', 'error', 'reaped'.
#[sea_orm(column_type = "Text")]
pub state: String,
#[sea_orm(default_value = 0)]

2
src/ipc.rs
View File

@@ -69,7 +69,7 @@ impl Event {
*bytes as f64 / 1_048_576.0
),
Event::Status { feeds, pending, downloaded } => {
format!("{feeds} feeds, {pending} pending, {downloaded} downloaded")
format!("{feeds} feeds, {pending} queued to download, {downloaded} downloaded")
}
Event::Error { msg } => format!("error: {msg}"),
// Noise in a terminal; a UI still gets them on the socket.

8
src/main.rs
View File

@@ -1333,6 +1333,10 @@ async fn scan_one(
// The body the scan already fetched for it, if it did (#104).
prefetched: Option<Result<feed::Fetched>>,
) -> Result<Outcome> {
// The queue follows the feed's settings each time it is due, changed or not, so files a
// limit or auto-download no longer reaches stop counting as waiting (#113).
let policy = policy_for(ctx, id, feed_cfg).await?;
ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?;
// A Patreon creator with more than one show is a list of feeds, like an OPML.
if feed::is_patreon_creator(&feed_cfg.url) {
match feed::patreon_shows(&ctx.client, &feed_cfg.url).await {
@@ -1456,7 +1460,6 @@ async fn scan_one(
parsed.category.as_deref(),
).await?;
let policy = policy_for(ctx, id, feed_cfg).await?;
if let Some(parent) = &feed_cfg.group {
let listed: Vec<(&str, &str)> = parsed
.entries
@@ -1503,6 +1506,9 @@ async fn scan_one(
if scan.new_entries > 0 {
ctx.db.rehide(id).await?; // what is new may hold someone's blocked words
}
// A new item takes a place among the newest and the oldest of them leaves the queue; a file a
// filter lets through again may be outside them.
ctx.db.hold_back(id, if policy.auto_download { policy.budget } else { 0 }).await?;
let budget = policy.budget;
if policy.auto_download && budget > 0 {

2
web/src/items.ts
View File

@@ -328,7 +328,7 @@ function encBox(x){
// Nothing on disk. For an image or a PDF you usually just want to look at it, so link
// straight to the publisher's copy in a new tab -- no download, and nothing proxied
// through here, which would make ipx a fetch-anything relay.
const viewable = !isPlayable(x) && x.state !== 'pending';
const viewable = !isPlayable(x) && x.state !== 'pending' && x.state !== 'held';
return `<div class="encbox">
${kindIcon(x)}
<span class="meta" style="flex:1">${size}</span>