Step 4: enclosure downloads

Streaming download with progress, content sniffing that replaces the
long-dead detectFileType(), HTML-body rejection, and placement by rename
from an incomplete dir on the same filesystem.

Filters are applied once at discovery and recorded in enclosures.state,
so the download queue is the table rather than the parse result and
max_new_per_check defers work instead of dropping it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RPyeapneuXrCdojsaiXGbe

src/main.rs

@@ -1,5 +1,6 @@
 mod config;
 mod db;
+mod download;
 mod feed;
 
 use anyhow::Result;
@@ -91,10 +92,11 @@ async fn fetch(
             }
         }
 
-        match scan_one(&client, db, id, feed_cfg, &state).await {
-            Ok(Some((new_entries, new_encs))) => {
-                println!("{id}: {new_entries} new entries, {new_encs} new enclosures")
-            }
+        match scan_one(&client, cfg, db, id, feed_cfg, &state).await {
+            Ok(Some(s)) => println!(
+                "{id}: {} new entries, {} downloaded, {} failed, {} torrents deferred",
+                s.new_entries, s.downloaded, s.failed, s.torrents
+            ),
             Ok(None) => println!("{id}: not modified"),
             Err(e) => {
                 // One bad feed must not end the scan.
@@ -110,11 +112,12 @@ async fn fetch(
 /// Ok(None) means 304.
 async fn scan_one(
     client: &reqwest::Client,
+    cfg: &config::Config,
     db: &db::Db,
     id: &str,
     feed_cfg: &config::Feed,
     state: &db::HttpState,
-) -> Result<Option<(usize, usize)>> {
+) -> Result<Option<Scan>> {
     let fetched = feed::fetch(
         client,
         feed_cfg,
@@ -141,19 +144,114 @@ async fn scan_one(
         parsed.ttl_mins,
     )?;
 
-    let mut new_entries = 0;
-    let mut new_encs = 0;
+    let mut scan = Scan::default();
     for entry in &parsed.entries {
         if db.record_entry(id, entry)? {
-            new_entries += 1;
+            scan.new_entries += 1;
         }
         for enc in &entry.enclosures {
-            if db.record_enclosure(id, &entry.guid, enc)? {
-                new_encs += 1;
+            if !db.record_enclosure(id, &entry.guid, enc)? {
+                continue; // Seen before: downloaded, skipped or deliberately reaped.
+            }
+            // Filters run once, at discovery, and are recorded in `state`. The download
+            // queue below is then just "everything still pending".
+            if let Some(reason) = reject(feed_cfg, entry, &enc.url) {
+                db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
             }
         }
     }
-    Ok(Some((new_entries, new_encs)))
+
+    let budget = feed_cfg.max_new_per_check.unwrap_or(usize::MAX);
+    if feed_cfg.auto_download && budget > 0 {
+        let folder = download::folder_for(cfg, id, feed_cfg, parsed.title.as_deref());
+        let dest_dir = cfg.general.download_dir.join(&folder);
+
+        for item in db.pending(id, budget)? {
+            if download::looks_like_torrent(&item.url, item.mime.as_deref()) {
+                // Step 7 owns these; recorded so nothing re-queues them meanwhile.
+                db.mark_enclosure(&item.url, "torrent", None)?;
+                scan.torrents += 1;
+                continue;
+            }
+            match fetch_one(client, cfg, db, feed_cfg, &item.url, &dest_dir).await {
+                Ok(path) => {
+                    println!("    saved {}", path.display());
+                    scan.downloaded += 1;
+                }
+                Err(e) => {
+                    let msg = format!("{e:#}");
+                    println!("    failed {}: {msg}", item.url);
+                    db.mark_enclosure(&item.url, "error", Some(&msg))?;
+                    scan.failed += 1;
+                }
+            }
+        }
+    }
+
+    Ok(Some(scan))
+}
+
+#[derive(Default)]
+struct Scan {
+    new_entries: usize,
+    downloaded: usize,
+    failed: usize,
+    torrents: usize,
+}
+
+/// Why this enclosure should not be downloaded, if it should not be.
+fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'static str> {
+    if !feed_cfg.auto_download {
+        return Some("auto_download is off");
+    }
+    if entry.explicit && !feed_cfg.allow_explicit {
+        return Some("explicit");
+    }
+    let categories = entry.categories.join(" ");
+    let haystacks = [
+        url,
+        entry.title.as_deref().unwrap_or(""),
+        entry.description.as_deref().unwrap_or(""),
+        categories.as_str(),
+    ];
+    if !download::matches_keywords(&feed_cfg.keywords, &haystacks) {
+        return Some("no keyword match");
+    }
+    None
+}
+
+async fn fetch_one(
+    client: &reqwest::Client,
+    cfg: &config::Config,
+    db: &db::Db,
+    feed_cfg: &config::Feed,
+    url: &str,
+    dest_dir: &std::path::Path,
+) -> Result<std::path::PathBuf> {
+    // Throttled to whole percents, as the original's lastDLStepSize guard did.
+    let mut last_pct = -1i64;
+    let name = download::filename_for(url, None);
+    let got = download::download(client, cfg, feed_cfg, url, |done, total| {
+        if let Some(t) = total.filter(|t| *t > 0) {
+            let pct = (done * 100 / t) as i64;
+            if pct > last_pct {
+                last_pct = pct;
+                println!("{}", download::progress_line(&name, done, total));
+            }
+        }
+    })
+    .await?;
+
+    if matches!(got.kind, download::Kind::Torrent) {
+        // The MIME lied. Don't file a .torrent as an episode.
+        let _ = tokio::fs::remove_file(&got.tmp).await;
+        db.mark_enclosure(url, "torrent", None)?;
+        anyhow::bail!("body is a torrent, deferred to the torrent downloader");
+    }
+
+    let path = download::place(&got, dest_dir).await?;
+    db.mark_downloaded(url, &path, got.bytes)?;
+    Ok(path)
 }
 
 fn ago(t: Option<i64>) -> String {