diff --git a/PROGRESS.md b/PROGRESS.md index 66305fe..1bfbb89 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -56,6 +56,30 @@ and until now nothing set them. --- +## 2026-09-10 — Image enclosures were being treated as episodes + +Reported as "Abort Retry Fail shows downloads but there are none". It had 20 enclosures, all +`image/jpeg`: Substack puts each article's header image in the RSS `` tag, so ipx +downloaded 20 JPEGs and counted them as episodes. Across the OPML subscription: 149 images, 74 MB, +11 feeds. + +`[general] media_types` defaults to `["audio", "video"]`, with a per-feed override. An enclosure +whose top-level type is not wanted is recorded and shown but not auto-downloaded. + +Clarified mid-change: those enclosures should still be *visible*, just not fetched automatically. +So nothing is hidden — the row names what it is ("image", "pdf", "torrent") instead of a bare +"skipped", and the download button still works if you want that file. + +Two judgement calls in `wanted_media`: an **unknown or absent** type is allowed, because the real +type is only known after downloading and refusing everything untyped would drop feeds that simply +omit the attribute; and a **torrent** is allowed, being a container rather than media, judged once +unpacked. + +The enclosure-less case Ray also described was already correct — every download affordance in the +row is gated on the enclosure existing. Checked before changing anything. + +--- + ## 2026-09-10 — Daemon I/O log tab, and Playwright **Log tabs.** All / Daemon I/O / Scans / HTTP. "Daemon I/O" is the control protocol itself: every diff --git a/README.md b/README.md index 004baae..009c980 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ organize = "feed" # "feed" | "date" max_total_gb = 50 # 0 = unlimited max_age_days = 30 # 0 = keep forever max_new_per_check = 3 # per feed, per scan. 0 = unlimited (pulls whole back catalogues) +media_types = ["audio", "video"] # what downloads automatically. Anything else is still + # listed and can be fetched by hand -- blog feeds put article + # images in enclosures. Empty takes everything. [torrent] enabled = true diff --git a/src/config.rs b/src/config.rs index 1f23e7d..3e300cf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -38,6 +38,10 @@ pub struct General { /// Unlimited by default was a trap: subscribing to an OPML of 80 feeds then pulled /// every back-catalogue episode at once. 0 means unlimited, deliberately chosen. pub max_new_per_check: usize, + /// Top-level media types worth downloading. Blog feeds put each article's header + /// image in an , so taking everything filled the disk with artwork and + /// counted it as episodes. Empty means take anything. + pub media_types: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] @@ -108,6 +112,9 @@ pub struct Feed { /// Overrides the global schedule for this feed. Same forms: "every 6h", "2d". #[serde(default, skip_serializing_if = "Option::is_none")] pub schedule: Option, + /// Media types for this feed. None follows `[general]`; an empty list takes anything. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_types: Option>, /// Cap on new downloads per scan for this feed. None follows `[general]`. #[serde(default, skip_serializing_if = "Option::is_none")] pub max_new_per_check: Option, @@ -135,6 +142,7 @@ impl Default for General { max_total_gb: 0.0, max_age_days: 0, max_new_per_check: 3, + media_types: vec!["audio".into(), "video".into()], } } } @@ -276,6 +284,25 @@ fn default_socket() -> PathBuf { } } +/// Whether an enclosure's type is one we want. +/// +/// An unknown type is allowed: the real type is only known after downloading, and +/// refusing everything untyped would drop feeds that simply omit the attribute. +pub fn wanted_media(mime: Option<&str>, wanted: &[String]) -> bool { + if wanted.is_empty() { + return true; + } + let Some(mime) = mime.map(str::trim).filter(|m| !m.is_empty()) else { + return true; + }; + let top = mime.split('/').next().unwrap_or(mime).to_ascii_lowercase(); + // A .torrent is a container for media, not media itself; judge it once unpacked. + if mime.to_ascii_lowercase().contains("torrent") { + return true; + } + wanted.iter().any(|w| w.trim().eq_ignore_ascii_case(&top) || w.trim().eq_ignore_ascii_case(mime)) +} + /// Feed ids are the TOML table key, so they must be readable and punctuation-free. pub fn slug(text: &str) -> String { let mut out = String::new(); @@ -391,6 +418,26 @@ mod tests { assert_eq!(t.ports(), (6881, 6889), "reversed range is not a range"); } + #[test] + fn media_types_keep_article_artwork_out() { + let want = vec!["audio".to_string(), "video".to_string()]; + assert!(wanted_media(Some("audio/mpeg"), &want)); + assert!(wanted_media(Some("audio/mp4"), &want)); + assert!(wanted_media(Some("video/quicktime"), &want)); + assert!(!wanted_media(Some("image/jpeg"), &want), "a blog header image is not an episode"); + assert!(!wanted_media(Some("text/html"), &want)); + + // A torrent is a container; what is inside is judged after unpacking. + assert!(wanted_media(Some("application/x-bittorrent"), &want)); + // Unknown type: only discoverable by downloading, so do not refuse it outright. + assert!(wanted_media(None, &want)); + assert!(wanted_media(Some(""), &want)); + // An empty list means take anything, which is how it behaved before. + assert!(wanted_media(Some("image/jpeg"), &[])); + // A full type can be named exactly. + assert!(wanted_media(Some("image/jpeg"), &["image/jpeg".to_string()])); + } + #[test] fn slugs_are_readable_and_unique() { assert_eq!(slug("Accidental Tech Podcast"), "accidental-tech-podcast"); @@ -402,7 +449,7 @@ mod tests { let mut taken = BTreeMap::new(); taken.insert("the-daily".to_string(), Feed { - url: "u".into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false, + url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, password: None, password_env: None, }); @@ -415,6 +462,7 @@ mod tests { url: "https://x/y".into(), folder: None, group: None, + media_types: None, schedule: None, keywords: vec![], allow_explicit: false, diff --git a/src/download.rs b/src/download.rs index f55221b..7b9e145 100644 --- a/src/download.rs +++ b/src/download.rs @@ -385,7 +385,7 @@ mod tests { let mut cfg = Config::default(); cfg.general.download_dir = "/tmp".into(); let mut f = crate::config::Feed { - url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, + url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, password: None, password_env: None, }; diff --git a/src/main.rs b/src/main.rs index 77fe69c..84d545a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -412,6 +412,7 @@ pub async fn add_one( url: url.to_owned(), folder: folder.clone(), group: None, + media_types: None, schedule: None, keywords: keywords.clone(), allow_explicit: false, @@ -494,6 +495,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path url, folder: None, group: None, + media_types: None, schedule: None, keywords: vec![], allow_explicit: false, @@ -711,6 +713,7 @@ pub fn subscriptions(ctx: &Ctx) -> Result> { url: m.url.clone(), folder: Some(format!("{base}/{title}")), group: Some(m.group_id.clone()), + media_types: parent.and_then(|p| p.media_types.clone()), schedule: parent.and_then(|p| p.schedule.clone()), keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(), allow_explicit: parent.is_some_and(|p| p.allow_explicit), @@ -837,7 +840,7 @@ async fn scan_one( } // Filters run once, at discovery, and are recorded in `state`. The download // queue below is then just "everything still pending". - if let Some(reason) = reject(feed_cfg, entry, &enc.url) { + if let Some(reason) = reject(&ctx.cfg(), feed_cfg, entry, enc) { ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?; } } @@ -991,10 +994,25 @@ async fn sync_opml( } /// Why this enclosure should not be downloaded, if it should not be. -fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'static str> { +fn reject( + cfg: &config::Config, + feed_cfg: &config::Feed, + entry: &feed::Entry, + enc: &feed::Enclosure, +) -> Option<&'static str> { + let url = enc.url.as_str(); if !feed_cfg.auto_download { return Some("auto_download is off"); } + // Blog feeds put the article's header image in an ; without this a text + // feed reads as a podcast full of episodes and fills the disk with artwork. + let wanted = feed_cfg + .media_types + .as_deref() + .unwrap_or(&cfg.general.media_types); + if !config::wanted_media(enc.mime.as_deref(), wanted) { + return Some("not a wanted media type"); + } if entry.explicit && !feed_cfg.allow_explicit { return Some("explicit"); } diff --git a/src/web.rs b/src/web.rs index 214e86b..be31a8e 100644 --- a/src/web.rs +++ b/src/web.rs @@ -283,7 +283,7 @@ mod tests { fn feed(url: &str) -> crate::config::Feed { crate::config::Feed { - url: url.into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false, + url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, password: None, password_env: None, } @@ -754,6 +754,7 @@ async fn import_opml( url, folder: None, group: None, + media_types: None, schedule: None, keywords: vec![], allow_explicit: false, @@ -776,6 +777,7 @@ struct Settings { schedule: String, every_mins: u64, max_new_per_check: usize, + media_types: Vec, download_dir: String, max_total_gb: f64, max_age_days: u64, @@ -787,6 +789,7 @@ async fn get_settings(State(state): State) -> Json { schedule: cfg.general.schedule.clone(), every_mins: cfg.general.interval(), max_new_per_check: cfg.general.max_new_per_check, + media_types: cfg.general.media_types.clone(), download_dir: cfg.general.download_dir.display().to_string(), max_total_gb: cfg.general.max_total_gb, max_age_days: cfg.general.max_age_days, @@ -797,6 +800,7 @@ async fn get_settings(State(state): State) -> Json { struct SettingsPatch { schedule: Option, max_new_per_check: Option, + media_types: Option>, max_total_gb: Option, max_age_days: Option, } @@ -820,6 +824,13 @@ async fn patch_settings( if let Some(v) = body.max_new_per_check { cfg.general.max_new_per_check = v; } + if let Some(v) = body.media_types { + cfg.general.media_types = v + .into_iter() + .map(|t| t.trim().to_lowercase()) + .filter(|t| !t.is_empty()) + .collect(); + } if let Some(v) = body.max_total_gb { cfg.general.max_total_gb = v.max(0.0); } diff --git a/web/index.html b/web/index.html index 0463908..ccbe9c6 100644 --- a/web/index.html +++ b/web/index.html @@ -606,7 +606,8 @@ function epEl(e){ ${num?`${num}`:''} ${dateOf(e.published)} ${left?''+left+'':''} - ${enc?`${has?'downloaded':esc(enc.state)}`:''} + ${enc?`${ + has?'downloaded':(enc.state==='skipped'?kindOf(enc):esc(enc.state))}`:''} ${enc&&enc.length?`${mb(enc.length)}`:''} ${e.flagged?'★ kept':''} @@ -615,7 +616,8 @@ function epEl(e){
${has?``: - (enc?``:'')} + (enc?``:'')} ${has?``:''} @@ -630,6 +632,18 @@ function epEl(e){ if(S.open.has(e.guid)) showNotes(e,el); return el; } +/// What an enclosure is, for a row that is not an episode: "image", "pdf", "document". +function kindOf(enc){ + const m=(enc.mime||'').toLowerCase(); + if(m.startsWith('image/')) return 'image'; + if(m.startsWith('video/')) return 'video'; + if(m.startsWith('audio/')) return 'audio'; + if(m.includes('pdf')) return 'pdf'; + if(m.includes('torrent')) return 'torrent'; + const ext=(enc.url||'').split('?')[0].split('.').pop(); + return (ext && ext.length<=5) ? ext.toLowerCase() : 'file'; +} + function feedArt(){ const f=S.feeds.find(x=>x.id===S.feed); return f&&f.image; } function toggleNotes(e,el){ @@ -937,6 +951,11 @@ async function prefsModal(){ Applies to any feed that does not set its own — including every feed inside an OPML subscription. 0 means unlimited, which will pull a whole back catalogue the first time a feed is scanned.
+
+ + Anything else is still listed and can be downloaded by hand — blog feeds + put article images in enclosures, and those are not episodes. Empty takes everything.
Over this, the oldest played episodes are deleted first. Starred @@ -952,6 +971,7 @@ async function prefsModal(){ await api('/api/settings',{method:'PATCH',body:JSON.stringify({ schedule:`every ${Math.max(1,Number($('#gnum').value)||1)}${$('#gunit').value}`, max_new_per_check:Math.max(0,Number($('#gmax').value)||0), + media_types:$('#gtypes').value.split(',').map(t=>t.trim()).filter(Boolean), max_total_gb:Number($('#gquota').value)||0, max_age_days:Number($('#gage').value)||0})}); closeModal(); toast('Settings saved');