rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SeaORM: enclosures, downloads and the reaper

Browse Source 
Twelve enclosure functions move to SeaORM: recording, the download queue,
marking done or failed, requeueing, and what the reaper may delete. INSERT OR
IGNORE becomes ON CONFLICT DO NOTHING; the reaper's read verdict is true or
false rather than 1 or 0, which Postgres would type as a 32-bit integer and
refuse to read as an i64; `read = 1` and `flagged = 1` test the booleans
themselves. retention::run and its callers (reap, rm, retire_group,
retire_stranded) become async.

The reaper deletes files, so it was checked on a copy of production against the
old SQL on the same file: all 2,195 candidates, identical and in the same order.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-18 18:38:07 +00:00
parent 6bf1ad6b31
commit a68bfb179b
4 changed files with 154 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

226
src/db.rs
View File

@@ -559,39 +559,46 @@ impl Db {
/// Returns true when this enclosure URL is new. False means we have downloaded it
/// before, or deliberately reaped it -- either way it is not fetched again.
pub fn mark_downloaded(&self, url: &str, path: &std::path::Path, bytes: u64) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE enclosures SET state = 'done', path = ?2, bytes_done = ?3,
downloaded_at = ?4, last_error = NULL WHERE url = ?1",
rusqlite::params![url, path.to_string_lossy(), bytes as i64, now()],
)?;
pub async fn mark_downloaded(&self, url: &str, path: &std::path::Path, bytes: u64) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = 'done', path = $2, bytes_done = $3,
downloaded_at = $4, last_error = NULL WHERE url = $1",
vec![url.into(), path.to_string_lossy().into_owned().into(), (bytes as i64).into(), now().into()],
)
.await?;
Ok(())
}
/// The row stays -- a failed URL is still a URL we have seen. `state` says why it has
/// no file, and a retry is an explicit act rather than something a rescan does silently.
pub fn mark_enclosure(&self, url: &str, state: &str, error: Option<&str>) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE enclosures SET state = ?2, last_error = ?3 WHERE url = ?1",
rusqlite::params![url, state, error],
)?;
pub async fn mark_enclosure(&self, url: &str, state: &str, error: Option<&str>) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = $2, last_error = $3 WHERE url = $1",
vec![url.into(), state.into(), error.map(str::to_owned).into()],
)
.await?;
Ok(())
}
pub fn record_enclosure(
pub async fn record_enclosure(
&self,
feed_id: &str,
guid: &str,
enc: &crate::feed::Enclosure,
) -> Result<bool> {
let conn = self.conn.lock().unwrap();
let inserted = conn.execute(
"INSERT OR IGNORE INTO enclosures (feed_id, guid, url, mime, length, state)
VALUES (?1, ?2, ?3, ?4, ?5, 'pending')",
rusqlite::params![feed_id, guid, enc.url, enc.mime, enc.length],
)?;
let inserted = self
.exec(
"INSERT INTO enclosures (feed_id, guid, url, mime, length, state)
VALUES ($1, $2, $3, $4, $5, 'pending') ON CONFLICT DO NOTHING",
vec![
feed_id.into(),
guid.into(),
enc.url.clone().into(),
enc.mime.clone().into(),
enc.length.into(),
],
)
.await?;
Ok(inserted == 1)
}
}
@@ -608,23 +615,21 @@ pub struct Pending {
impl Db {
/// The download queue is the table, not the parse result: an enclosure held back by
/// `max_new_per_check` is simply picked up by the next scan, in feed order.
pub fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
// Newest first: a cap of 3 should mean the three latest episodes, not the
// three that happen to have been recorded first.
pub async fn pending(&self, feed_id: &str, limit: usize) -> Result<Vec<Pending>> {
// Newest first: a cap of 3 should mean the three latest episodes, not the three that
// happen to have been recorded first.
self.rows(
"SELECT x.id, x.url, x.mime FROM enclosures x
JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid
WHERE x.feed_id = ?1 AND x.state = 'pending'
WHERE x.feed_id = $1 AND x.state = 'pending'
ORDER BY coalesce(e.published, e.first_seen) DESC, x.id DESC
LIMIT ?2",
)?;
let rows = stmt
.query_map(rusqlite::params![feed_id, limit as i64], |r| {
Ok(Pending { id: r.get(0)?, url: r.get(1)?, mime: r.get(2)? })
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
LIMIT $2",
vec![feed_id.into(), (limit as i64).into()],
)
.await?
.iter()
.map(|r| Ok(Pending { id: r.try_get("", "id")?, url: r.try_get("", "url")?, mime: r.try_get("", "mime")? }))
.collect()
}
}
@@ -652,57 +657,53 @@ impl Db {
///
/// (The Python intended `read = 1 AND flagged = 0` but never achieved it -- a missing
/// plistlib import and an `EntreiesData` typo meant the filter always threw.)
pub fn reap_candidates(&self) -> Result<Vec<Candidate>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
"SELECT e.id, e.url, e.path, e.bytes_done, coalesce(e.downloaded_at, 0),
CASE WHEN coalesce(readers.n, 0) >= coalesce(subs.n, 0) THEN 1 ELSE 0 END
pub async fn reap_candidates(&self) -> Result<Vec<Candidate>> {
// Yes/no as true and false, not 1 and 0: Postgres types a bare 1 as a 32-bit integer
// and will not hand it over as an i64.
self.rows(
"SELECT e.id, e.url, e.path, e.bytes_done, coalesce(e.downloaded_at, 0) AS age_key,
CASE WHEN coalesce(readers.n, 0) >= coalesce(subs.n, 0) THEN true ELSE false END AS read
FROM enclosures e
LEFT JOIN (SELECT feed_id, count(*) n FROM subscriptions GROUP BY feed_id) subs
LEFT JOIN (SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id) subs
ON subs.feed_id = e.feed_id
LEFT JOIN (SELECT feed_id, guid, count(*) n FROM entry_state
WHERE read = 1 GROUP BY feed_id, guid) readers
LEFT JOIN (SELECT feed_id, guid, count(*) AS n FROM entry_state
WHERE read GROUP BY feed_id, guid) readers
ON readers.feed_id = e.feed_id AND readers.guid = e.guid
WHERE e.path IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM entry_state s
WHERE s.feed_id = e.feed_id AND s.guid = e.guid AND s.flagged = 1)
WHERE s.feed_id = e.feed_id AND s.guid = e.guid AND s.flagged)
ORDER BY 6 DESC, coalesce(e.downloaded_at, 0) ASC, e.id ASC",
)?;
let rows = stmt
.query_map([], |r| {
Ok(Candidate {
id: r.get(0)?,
url: r.get(1)?,
path: r.get(2)?,
bytes: r.get(3)?,
age_key: r.get(4)?,
read: r.get::<_, i64>(5)? != 0,
})
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows)
vec![],
)
.await?
.iter()
.map(|r| {
Ok(Candidate {
id: r.try_get("", "id")?,
url: r.try_get("", "url")?,
path: r.try_get("", "path")?,
bytes: r.try_get("", "bytes_done")?,
age_key: r.try_get("", "age_key")?,
read: r.try_get("", "read")?,
})
})
.collect()
}
/// The row survives the file: that is what stops a reaped episode being re-downloaded.
pub fn mark_reaped(&self, id: i64) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE enclosures SET state = 'reaped', path = NULL WHERE id = ?1",
[id],
)?;
pub async fn mark_reaped(&self, id: i64) -> Result<()> {
self.exec("UPDATE enclosures SET state = 'reaped', path = NULL WHERE id = $1", vec![id.into()]).await?;
Ok(())
}
/// Rows claiming a file that is no longer there (someone deleted it by hand).
pub fn missing_files(&self) -> Result<Vec<(i64, String)>> {
let conn = self.conn.lock().unwrap();
let mut stmt =
conn.prepare("SELECT id, path FROM enclosures WHERE path IS NOT NULL")?;
let rows = stmt
.query_map([], |r| Ok((r.get(0)?, r.get::<_, String>(1)?)))?
.collect::<rusqlite::Result<Vec<_>>>()?;
Ok(rows
pub async fn missing_files(&self) -> Result<Vec<(i64, String)>> {
Ok(enclosures::Entity::find()
.filter(enclosures::Column::Path.is_not_null())
.all(&self.orm)
.await?
.into_iter()
.filter_map(|e| Some((e.id, e.path?)))
.filter(|(_, p)| !std::path::Path::new(p).exists())
.collect())
}
@@ -1401,42 +1402,23 @@ impl Db {
/// The next N enclosures with no file, newest entry first -- what "download latest"
/// queues up.
pub fn undownloaded(&self, feed_id: &str, limit: i64) -> Result<Vec<i64>> {
let conn = self.conn.lock().unwrap();
let mut stmt = conn.prepare(
pub async fn undownloaded(&self, feed_id: &str, limit: i64) -> Result<Vec<i64>> {
self.rows(
"SELECT x.id FROM enclosures x
JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid
WHERE x.feed_id = ?1 AND x.path IS NULL AND x.state != 'reaped'
WHERE x.feed_id = $1 AND x.path IS NULL AND x.state <> 'reaped'
ORDER BY coalesce(e.published, e.first_seen) DESC
LIMIT ?2",
)?;
Ok(stmt
.query_map(rusqlite::params![feed_id, limit], |r| r.get(0))?
.collect::<rusqlite::Result<Vec<_>>>()?)
LIMIT $2",
vec![feed_id.into(), limit.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "id")?))
.collect()
}
pub fn enclosure(&self, id: i64) -> Result<Option<EncRow>> {
let conn = self.conn.lock().unwrap();
Ok(conn
.query_row(
"SELECT id, feed_id, guid, url, mime, length, path, state, last_error
FROM enclosures WHERE id = ?1",
[id],
|r| {
Ok(EncRow {
id: r.get(0)?,
feed_id: r.get(1)?,
guid: r.get(2)?,
url: r.get(3)?,
mime: r.get(4)?,
length: r.get(5)?,
path: r.get(6)?,
state: r.get(7)?,
last_error: r.get(8)?,
})
},
)
.optional()?)
pub async fn enclosure(&self, id: i64) -> Result<Option<EncRow>> {
Ok(enclosures::Entity::find_by_id(id).one(&self.orm).await?.map(EncRow::from))
}
/// Read and kept, per person. The row is created on first touch.
@@ -1466,13 +1448,12 @@ impl Db {
/// How many files this feed has on disk. Decides whether a feed dropped from an OPML
/// can be removed or must be kept.
pub fn downloaded_count(&self, feed_id: &str) -> Result<i64> {
let conn = self.conn.lock().unwrap();
Ok(conn.query_row(
"SELECT count(*) FROM enclosures WHERE feed_id = ?1 AND path IS NOT NULL",
[feed_id],
|r| r.get(0),
)?)
pub async fn downloaded_count(&self, feed_id: &str) -> Result<i64> {
Ok(enclosures::Entity::find()
.filter(enclosures::Column::FeedId.eq(feed_id))
.filter(enclosures::Column::Path.is_not_null())
.count(&self.orm)
.await? as i64)
}
/// Names a feed without touching its conditional-GET validators. An OPML subscription
@@ -1675,23 +1656,20 @@ impl Db {
/// Nothing can be in flight the moment the daemon starts, so any row still marked
/// `downloading` is a leftover from a restart or a crash. Left alone it would sit
/// there forever: the pending queue skips it and nothing else ever revisits it.
pub fn requeue_interrupted(&self) -> Result<usize> {
let conn = self.conn.lock().unwrap();
Ok(conn.execute(
"UPDATE enclosures SET state = 'pending' WHERE state = 'downloading' AND path IS NULL",
[],
)?)
pub async fn requeue_interrupted(&self) -> Result<usize> {
Ok(self
.exec("UPDATE enclosures SET state = 'pending' WHERE state = 'downloading' AND path IS NULL", vec![])
.await? as usize)
}
/// Puts an enclosure back in the queue so the next scan picks it up. This is how a
/// `skipped` verdict (from a filter that has since been changed) gets revisited.
pub fn requeue(&self, id: i64) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"UPDATE enclosures SET state = 'pending', last_error = NULL
WHERE id = ?1 AND path IS NULL",
[id],
)?;
pub async fn requeue(&self, id: i64) -> Result<()> {
self.exec(
"UPDATE enclosures SET state = 'pending', last_error = NULL WHERE id = $1 AND path IS NULL",
vec![id.into()],
)
.await?;
Ok(())
}
}
@@ -2085,7 +2063,7 @@ mod tests {
(3,'f','new','u-new','pending');",
)
.unwrap();
let got: Vec<String> = db.pending("f", 2).unwrap().into_iter().map(|p| p.url).collect();
let got: Vec<String> = db.pending("f", 2).await.unwrap().into_iter().map(|p| p.url).collect();
assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later");
}
@@ -2100,7 +2078,7 @@ mod tests {
(4,'f','d','u4','done','/tmp/x');",
)
.unwrap();
assert_eq!(db.requeue_interrupted().unwrap(), 1, "only the in-flight, fileless one");
assert_eq!(db.requeue_interrupted().await.unwrap(), 1, "only the in-flight, fileless one");
let conn = db.conn.lock().unwrap();
let state = |id: i64| -> String {
conn.query_row("SELECT state FROM enclosures WHERE id = ?1", [id], |r| r.get(0)).unwrap()