rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SeaORM: enclosures, downloads and the reaper

Browse Source 
Twelve enclosure functions move to SeaORM: recording, the download queue,
marking done or failed, requeueing, and what the reaper may delete. INSERT OR
IGNORE becomes ON CONFLICT DO NOTHING; the reaper's read verdict is true or
false rather than 1 or 0, which Postgres would type as a 32-bit integer and
refuse to read as an i64; `read = 1` and `flagged = 1` test the booleans
themselves. retention::run and its callers (reap, rm, retire_group,
retire_stranded) become async.

The reaper deletes files, so it was checked on a copy of production against the
old SQL on the same file: all 2,195 candidates, identical and in the same order.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-09-18 18:38:07 +00:00
parent 6bf1ad6b31
commit a68bfb179b
4 changed files with 154 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
src/main.rs
View File

@@ -225,7 +225,7 @@ async fn main() -> Result<()> {
Command::Add { url, folder, keywords } => {
add(&ctx, &config_path, &url, folder, keywords).await
}
Command::Rm { feed } => rm(&ctx, &config_path, &feed),
Command::Rm { feed } => rm(&ctx, &config_path, &feed).await,
Command::User { cmd } => user_cmd(&ctx, cmd).await,
Command::Import { file } => import(&ctx, &config_path, &file).await,
Command::Export { file } => export(&ctx, &file),
@@ -334,10 +334,10 @@ async fn run(ctx: &Arc<Ctx>, cmd: Cmd) -> Result<()> {
match cmd {
Cmd::Fetch { feed, force } => {
// Make room before pulling more down, as the original did per download.
reap(ctx, false, false)?;
reap(ctx, false, false).await?;
fetch(ctx, feed.as_deref(), force).await
}
Cmd::Reap { dry_run } => reap(ctx, dry_run, true),
Cmd::Reap { dry_run } => reap(ctx, dry_run, true).await,
Cmd::Download { enclosure } => download_one(ctx, enclosure).await,
Cmd::Status => {
ctx.out.emit(status(ctx).await);
@@ -387,13 +387,13 @@ async fn daemon(
}
}
match ctx.db.requeue_interrupted() {
match ctx.db.requeue_interrupted().await {
Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"),
Ok(_) => {}
Err(e) => tracing::warn!(error = ?e, "could not requeue interrupted downloads"),
}
match retire_stranded(&ctx) {
match retire_stranded(&ctx).await {
Ok(0) => {}
Ok(n) => tracing::info!(feeds = n, "retired feeds whose OPML is no longer in config"),
Err(e) => tracing::warn!(error = ?e, "could not retire feeds whose OPML is no longer in config"),
@@ -651,7 +651,7 @@ fn url_stem(url: &str) -> String {
.unwrap_or_else(|| url.to_owned())
}
fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> {
async fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> {
let mut cfg = (*ctx.cfg()).clone();
if cfg.feeds.remove(feed).is_none() {
// Derived from an OPML: drop it here, though the subscription will list it again
@@ -663,7 +663,7 @@ fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> {
cfg.save(config_path)?;
// State and files stay: re-adding the feed should not re-download its back catalogue.
println!("removed {feed}; downloads and history kept");
retire_group(ctx, feed)?;
retire_group(ctx, feed).await?;
Ok(())
}
@@ -813,8 +813,8 @@ fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> {
/// `standalone` false means this is the sweep that runs before a scan: it reports what it
/// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch`
/// would stop reading before the scan had even started.
fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
let r = retention::run(&ctx.cfg(), &ctx.db, dry_run)?;
async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?;
for c in r.aged_out.iter().chain(r.over_quota.iter()) {
ctx.out.emit(Event::Reaped {
path: c.path.clone(),
@@ -992,14 +992,14 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
/// itself is removed, since `subscriptions()` would otherwise keep scanning them under a
/// fallback policy meant for a feed with no parent at all. A feed promoted to config is not
/// derived any more, so it is only unmanaged.
pub fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
let cfg = ctx.cfg();
for m in ctx.db.managed_feeds()?.into_iter().filter(|m| m.group_id == parent_id) {
if cfg.feeds.contains_key(&m.id) {
// Scanned from its config entry and still read. Dropped as derived, its stored
// entries would go with it: davewiner's 11 were promoted without being unmanaged.
ctx.db.unmanage(&m.id)?;
} else if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 {
} else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
ctx.db.set_orphaned(&m.id, true)?;
} else {
ctx.db.drop_managed(&m.id)?;
@@ -1012,7 +1012,7 @@ pub fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
/// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind:
/// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were
/// most of the ones stored.
fn retire_stranded(ctx: &Ctx) -> Result<usize> {
async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
let cfg = ctx.cfg();
let before = ctx.db.managed_feeds()?;
let stranded: std::collections::BTreeSet<&str> = before
@@ -1021,7 +1021,7 @@ fn retire_stranded(ctx: &Ctx) -> Result<usize> {
.filter(|g| !cfg.feeds.contains_key(*g))
.collect();
for group in stranded {
retire_group(ctx, group)?;
retire_group(ctx, group).await?;
}
Ok(before.len() - ctx.db.managed_feeds()?.len())
}
@@ -1159,7 +1159,7 @@ async fn scan_one(
scan.new_entries += 1;
}
for enc in &entry.enclosures {
let was = if ctx.db.record_enclosure(id, &entry.guid, enc)? {
let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())
@@ -1169,8 +1169,8 @@ async fn scan_one(
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
if now != was {
match now {
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None)?,
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?,
}
}
}
@@ -1182,10 +1182,10 @@ async fn scan_one(
let folder = download::folder_for(&cfg, id, feed_cfg, parsed.title.as_deref());
let dest_dir = cfg.general.download_dir.join(&folder);
for item in ctx.db.pending(id, budget)? {
for item in ctx.db.pending(id, budget).await? {
if download::looks_like_torrent(&item.url, item.mime.as_deref()) {
if !ctx.cfg().torrent.enabled {
ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled"))?;
ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled")).await?;
ctx.out.emit(Event::TorrentDeferred {
feed: id.to_string(),
url: item.url.clone(),
@@ -1195,14 +1195,14 @@ async fn scan_one(
}
if ctx.detach_torrents {
// 'downloading' keeps the next scan from queueing it a second time.
ctx.db.mark_enclosure(&item.url, "downloading", None)?;
ctx.db.mark_enclosure(&item.url, "downloading", None).await?;
spawn_torrent(ctx, id.to_string(), item.id, item.url.clone(), dest_dir.clone());
scan.torrents += 1;
continue;
}
match torrent_one(ctx, id, item.id, &item.url, &dest_dir).await {
Ok((path, bytes)) => {
ctx.db.mark_downloaded(&item.url, &path, bytes)?;
ctx.db.mark_downloaded(&item.url, &path, bytes).await?;
ctx.out.emit(Event::DownloadDone {
feed: id.to_string(),
enclosure: item.id,
@@ -1220,7 +1220,7 @@ async fn scan_one(
url: item.url.clone(),
msg: msg.clone(),
});
ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?;
ctx.db.mark_enclosure(&item.url, "error", Some(&msg)).await?;
scan.failed += 1;
}
}
@@ -1245,7 +1245,7 @@ async fn scan_one(
url: item.url.clone(),
msg: msg.clone(),
});
ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?;
ctx.db.mark_enclosure(&item.url, "error", Some(&msg)).await?;
scan.failed += 1;
}
}
@@ -1334,7 +1334,7 @@ async fn sync_group(
if listed.iter().any(|(_, u)| u == &m.url) {
continue;
}
if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 {
if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
// Never orphan a downloaded file: keep the feed and say why in the UI.
ctx.db.set_orphaned(&m.id, true)?;
kept += 1;
@@ -1488,14 +1488,14 @@ async fn fetch_one(
// as if it were an episode.
let _ = tokio::fs::remove_file(&got.tmp).await;
if !ctx.cfg().torrent.enabled {
ctx.db.mark_enclosure(url, "skipped", Some("torrents disabled"))?;
ctx.db.mark_enclosure(url, "skipped", Some("torrents disabled")).await?;
anyhow::bail!("body is a torrent and torrents are disabled");
}
return torrent_one(ctx, feed_id, enclosure, url, dest_dir).await;
}
let path = download::place(&got, dest_dir).await?;
ctx.db.mark_downloaded(url, &path, got.bytes)?;
ctx.db.mark_downloaded(url, &path, got.bytes).await?;
Ok((path, got.bytes))
}
@@ -1513,7 +1513,7 @@ fn spawn_torrent(ctx: &Arc<Ctx>, feed_id: String, enclosure: i64, url: String, d
let db = &ctx.db;
match outcome {
Ok((path, bytes)) => {
if let Err(e) = db.mark_downloaded(&url, &path, bytes) {
if let Err(e) = db.mark_downloaded(&url, &path, bytes).await {
tracing::warn!(error = ?e, "could not record the finished torrent");
}
ctx.out.emit(Event::DownloadDone {
@@ -1526,7 +1526,7 @@ fn spawn_torrent(ctx: &Arc<Ctx>, feed_id: String, enclosure: i64, url: String, d
}
Err(e) => {
let msg = format!("{e:#}");
let _ = db.mark_enclosure(&url, "error", Some(&msg));
let _ = db.mark_enclosure(&url, "error", Some(&msg)).await;
ctx.out.emit(Event::DownloadError { feed: feed_id, enclosure, url, msg });
}
}
@@ -1539,7 +1539,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
let cfg = ctx.cfg();
let enc = ctx
.db
.enclosure(id)?
.enclosure(id).await?
.ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?;
if enc.path.is_some() {
return Ok(()); // Already here.
@@ -1561,7 +1561,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
ctx.out.emit(Event::FeedStart { feed: enc.feed_id.clone() });
let is_torrent = download::looks_like_torrent(&enc.url, enc.mime.as_deref());
if is_torrent && cfg.torrent.enabled && ctx.detach_torrents {
ctx.db.mark_enclosure(&enc.url, "downloading", None)?;
ctx.db.mark_enclosure(&enc.url, "downloading", None).await?;
spawn_torrent(ctx, enc.feed_id.clone(), enc.id, enc.url.clone(), dest_dir);
return Ok(());
}
@@ -1577,7 +1577,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
match result {
Ok((path, bytes)) => {
ctx.db.mark_downloaded(&enc.url, &path, bytes)?;
ctx.db.mark_downloaded(&enc.url, &path, bytes).await?;
ctx.out.emit(Event::DownloadDone {
feed: enc.feed_id.clone(),
enclosure: enc.id,
@@ -1588,7 +1588,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
}
Err(e) => {
let msg = format!("{e:#}");
ctx.db.mark_enclosure(&enc.url, "error", Some(&msg))?;
ctx.db.mark_enclosure(&enc.url, "error", Some(&msg)).await?;
ctx.out.emit(Event::DownloadError {
feed: enc.feed_id.clone(),
enclosure: enc.id,
@@ -1756,10 +1756,10 @@ mod tests {
ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap();
ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap();
let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None };
ctx.db.record_enclosure("has-file", "g1", &enc).unwrap();
ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).unwrap();
ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap();
ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap();
retire_group(&ctx, "parent").unwrap();
retire_group(&ctx, "parent").await.unwrap();
let managed = ctx.db.managed_feeds().unwrap();
assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten");
@@ -1780,7 +1780,7 @@ mod tests {
ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").unwrap();
ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).unwrap();
assert_eq!(retire_stranded(&ctx).unwrap(), 2, "empty dropped, promoted unmanaged");
assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged");
let managed: Vec<String> = ctx.db.managed_feeds().unwrap().into_iter().map(|m| m.id).collect();
assert_eq!(managed, ["listed"], "a group still in config is left alone");