Count as queued only what a scan will download on its own (#113)
Every file in state 'pending' counted as waiting to download: 4420 in production, across 12 shows. Since #97 a scan only downloads among a feed's newest max_new_per_check items, so those were back-catalogue episodes no scan would take; the real queue was 0. A new state, 'held': listed and downloadable by hand, but outside the feed's newest items, or of a feed nothing downloads automatically. Db::hold_back moves a feed's waiting files between 'pending' and 'held' each time the feed is due, changed or not, and again after its items are stored, so a new episode, a raised limit or auto-download turned on or off moves them. A held file keeps its item's place among the newest, as a downloaded one does. 'pending' now means queued, so ipx status, /api/status and the dashboard read true without changing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
56
src/db.rs
56
src/db.rs
@@ -652,6 +652,35 @@ impl Db {
|
||||
/// through every show's history: 4420 files queued across 12 shows on the default of 3
|
||||
/// (#97). Unlimited, 0 in the settings, is the whole back catalogue, for an archive. An item
|
||||
/// whose files were all skipped by a filter does not hold one of the places.
|
||||
/// Puts a feed's waiting files in the queue or out of it: 'pending' for those among its
|
||||
/// `limit` newest items, which the scan downloads on its own, and 'held' for the rest, listed
|
||||
/// to download by hand. With all of them 'pending', the queue counted 4420 back-catalogue
|
||||
/// files no scan would ever take (#97, #113). 0 holds them all, for a feed nobody downloads
|
||||
/// automatically. A held file still holds its item's place among the newest.
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn hold_back(&self, feed_id: &str, limit: usize) -> Result<()> {
|
||||
let newest = "SELECT n.guid FROM entries n
|
||||
WHERE n.feed_id = $1
|
||||
AND EXISTS (SELECT 1 FROM enclosures y
|
||||
WHERE y.feed_id = n.feed_id AND y.guid = n.guid
|
||||
AND y.state <> 'skipped')
|
||||
ORDER BY coalesce(n.published, n.first_seen) DESC, n.guid DESC
|
||||
LIMIT $2";
|
||||
let limit: sea_orm::Value = (limit.min(i64::MAX as usize) as i64).into();
|
||||
self.exec(
|
||||
&format!("UPDATE enclosures SET state = 'held' WHERE feed_id = $1 AND state = 'pending' AND guid NOT IN ({newest})"),
|
||||
vec![feed_id.into(), limit.clone()],
|
||||
)
|
||||
.await?;
|
||||
// And back, when a limit is raised or a feed downloads again.
|
||||
self.exec(
|
||||
&format!("UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"),
|
||||
vec![feed_id.into(), limit],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
pub async fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
|
||||
self.rows(
|
||||
@@ -2130,6 +2159,33 @@ mod tests {
|
||||
assert!(db.prune_abandoned_failures().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_the_newest_files_stay_queued_and_the_rest_are_held() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
|
||||
('f','e1',0,100),('f','e2',0,200),('f','e3',0,300),('f','e4',0,400);
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
|
||||
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
|
||||
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);",
|
||||
).await
|
||||
.unwrap();
|
||||
let states = || async {
|
||||
let mut v: Vec<(i64, String)> = db.rows("SELECT id, state FROM enclosures ORDER BY id", vec![]).await.unwrap()
|
||||
.iter().map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap())).collect();
|
||||
v.sort();
|
||||
v.into_iter().map(|(_, s)| s).collect::<Vec<_>>()
|
||||
};
|
||||
db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3
|
||||
assert_eq!(states().await, ["held", "held", "done", "pending"]);
|
||||
db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back
|
||||
assert_eq!(states().await, ["held", "pending", "done", "pending"]);
|
||||
db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held
|
||||
assert_eq!(states().await, ["held", "held", "done", "held"]);
|
||||
db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued
|
||||
assert_eq!(states().await, ["pending", "pending", "done", "pending"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_limit_takes_the_newest_episodes_not_the_back_catalogue() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user