Do not auto-download enclosures that are not audio or video
Blog feeds put each article's header image in an <enclosure>, so a text feed read as a podcast full of episodes: 149 images, 74 MB across 11 feeds. media_types defaults to audio and video, with a per-feed override. Such enclosures stay listed and stay downloadable by hand; the row names what it is rather than saying "skipped". An unknown type is allowed, since the real type is only known after downloading, and a torrent is allowed as a container judged once unpacked. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
This commit is contained in:
24
PROGRESS.md
24
PROGRESS.md
@@ -56,6 +56,30 @@ and until now nothing set them.
|
||||
|
||||
---
|
||||
|
||||
## 2026-09-10 — Image enclosures were being treated as episodes
|
||||
|
||||
Reported as "Abort Retry Fail shows downloads but there are none". It had 20 enclosures, all
|
||||
`image/jpeg`: Substack puts each article's header image in the RSS `<enclosure>` tag, so ipx
|
||||
downloaded 20 JPEGs and counted them as episodes. Across the OPML subscription: 149 images, 74 MB,
|
||||
11 feeds.
|
||||
|
||||
`[general] media_types` defaults to `["audio", "video"]`, with a per-feed override. An enclosure
|
||||
whose top-level type is not wanted is recorded and shown but not auto-downloaded.
|
||||
|
||||
Clarified mid-change: those enclosures should still be *visible*, just not fetched automatically.
|
||||
So nothing is hidden — the row names what it is ("image", "pdf", "torrent") instead of a bare
|
||||
"skipped", and the download button still works if you want that file.
|
||||
|
||||
Two judgement calls in `wanted_media`: an **unknown or absent** type is allowed, because the real
|
||||
type is only known after downloading and refusing everything untyped would drop feeds that simply
|
||||
omit the attribute; and a **torrent** is allowed, being a container rather than media, judged once
|
||||
unpacked.
|
||||
|
||||
The enclosure-less case Ray also described was already correct — every download affordance in the
|
||||
row is gated on the enclosure existing. Checked before changing anything.
|
||||
|
||||
---
|
||||
|
||||
## 2026-09-10 — Daemon I/O log tab, and Playwright
|
||||
|
||||
**Log tabs.** All / Daemon I/O / Scans / HTTP. "Daemon I/O" is the control protocol itself: every
|
||||
|
||||
@@ -59,6 +59,9 @@ organize = "feed" # "feed" | "date"
|
||||
max_total_gb = 50 # 0 = unlimited
|
||||
max_age_days = 30 # 0 = keep forever
|
||||
max_new_per_check = 3 # per feed, per scan. 0 = unlimited (pulls whole back catalogues)
|
||||
media_types = ["audio", "video"] # what downloads automatically. Anything else is still
|
||||
# listed and can be fetched by hand -- blog feeds put article
|
||||
# images in enclosures. Empty takes everything.
|
||||
|
||||
[torrent]
|
||||
enabled = true
|
||||
|
||||
@@ -38,6 +38,10 @@ pub struct General {
|
||||
/// Unlimited by default was a trap: subscribing to an OPML of 80 feeds then pulled
|
||||
/// every back-catalogue episode at once. 0 means unlimited, deliberately chosen.
|
||||
pub max_new_per_check: usize,
|
||||
/// Top-level media types worth downloading. Blog feeds put each article's header
|
||||
/// image in an <enclosure>, so taking everything filled the disk with artwork and
|
||||
/// counted it as episodes. Empty means take anything.
|
||||
pub media_types: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
|
||||
@@ -108,6 +112,9 @@ pub struct Feed {
|
||||
/// Overrides the global schedule for this feed. Same forms: "every 6h", "2d".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub schedule: Option<String>,
|
||||
/// Media types for this feed. None follows `[general]`; an empty list takes anything.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub media_types: Option<Vec<String>>,
|
||||
/// Cap on new downloads per scan for this feed. None follows `[general]`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_new_per_check: Option<usize>,
|
||||
@@ -135,6 +142,7 @@ impl Default for General {
|
||||
max_total_gb: 0.0,
|
||||
max_age_days: 0,
|
||||
max_new_per_check: 3,
|
||||
media_types: vec!["audio".into(), "video".into()],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -276,6 +284,25 @@ fn default_socket() -> PathBuf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an enclosure's type is one we want.
|
||||
///
|
||||
/// An unknown type is allowed: the real type is only known after downloading, and
|
||||
/// refusing everything untyped would drop feeds that simply omit the attribute.
|
||||
pub fn wanted_media(mime: Option<&str>, wanted: &[String]) -> bool {
|
||||
if wanted.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let Some(mime) = mime.map(str::trim).filter(|m| !m.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let top = mime.split('/').next().unwrap_or(mime).to_ascii_lowercase();
|
||||
// A .torrent is a container for media, not media itself; judge it once unpacked.
|
||||
if mime.to_ascii_lowercase().contains("torrent") {
|
||||
return true;
|
||||
}
|
||||
wanted.iter().any(|w| w.trim().eq_ignore_ascii_case(&top) || w.trim().eq_ignore_ascii_case(mime))
|
||||
}
|
||||
|
||||
/// Feed ids are the TOML table key, so they must be readable and punctuation-free.
|
||||
pub fn slug(text: &str) -> String {
|
||||
let mut out = String::new();
|
||||
@@ -391,6 +418,26 @@ mod tests {
|
||||
assert_eq!(t.ports(), (6881, 6889), "reversed range is not a range");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_types_keep_article_artwork_out() {
|
||||
let want = vec!["audio".to_string(), "video".to_string()];
|
||||
assert!(wanted_media(Some("audio/mpeg"), &want));
|
||||
assert!(wanted_media(Some("audio/mp4"), &want));
|
||||
assert!(wanted_media(Some("video/quicktime"), &want));
|
||||
assert!(!wanted_media(Some("image/jpeg"), &want), "a blog header image is not an episode");
|
||||
assert!(!wanted_media(Some("text/html"), &want));
|
||||
|
||||
// A torrent is a container; what is inside is judged after unpacking.
|
||||
assert!(wanted_media(Some("application/x-bittorrent"), &want));
|
||||
// Unknown type: only discoverable by downloading, so do not refuse it outright.
|
||||
assert!(wanted_media(None, &want));
|
||||
assert!(wanted_media(Some(""), &want));
|
||||
// An empty list means take anything, which is how it behaved before.
|
||||
assert!(wanted_media(Some("image/jpeg"), &[]));
|
||||
// A full type can be named exactly.
|
||||
assert!(wanted_media(Some("image/jpeg"), &["image/jpeg".to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slugs_are_readable_and_unique() {
|
||||
assert_eq!(slug("Accidental Tech Podcast"), "accidental-tech-podcast");
|
||||
@@ -402,7 +449,7 @@ mod tests {
|
||||
|
||||
let mut taken = BTreeMap::new();
|
||||
taken.insert("the-daily".to_string(), Feed {
|
||||
url: "u".into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
auto_download: true, max_new_per_check: None, username: None,
|
||||
password: None, password_env: None,
|
||||
});
|
||||
@@ -415,6 +462,7 @@ mod tests {
|
||||
url: "https://x/y".into(),
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
|
||||
@@ -385,7 +385,7 @@ mod tests {
|
||||
let mut cfg = Config::default();
|
||||
cfg.general.download_dir = "/tmp".into();
|
||||
let mut f = crate::config::Feed {
|
||||
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None,
|
||||
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
|
||||
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
|
||||
max_new_per_check: None, username: None, password: None, password_env: None,
|
||||
};
|
||||
|
||||
22
src/main.rs
22
src/main.rs
@@ -412,6 +412,7 @@ pub async fn add_one(
|
||||
url: url.to_owned(),
|
||||
folder: folder.clone(),
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: keywords.clone(),
|
||||
allow_explicit: false,
|
||||
@@ -494,6 +495,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path
|
||||
url,
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
@@ -711,6 +713,7 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
|
||||
url: m.url.clone(),
|
||||
folder: Some(format!("{base}/{title}")),
|
||||
group: Some(m.group_id.clone()),
|
||||
media_types: parent.and_then(|p| p.media_types.clone()),
|
||||
schedule: parent.and_then(|p| p.schedule.clone()),
|
||||
keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(),
|
||||
allow_explicit: parent.is_some_and(|p| p.allow_explicit),
|
||||
@@ -837,7 +840,7 @@ async fn scan_one(
|
||||
}
|
||||
// Filters run once, at discovery, and are recorded in `state`. The download
|
||||
// queue below is then just "everything still pending".
|
||||
if let Some(reason) = reject(feed_cfg, entry, &enc.url) {
|
||||
if let Some(reason) = reject(&ctx.cfg(), feed_cfg, entry, enc) {
|
||||
ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
|
||||
}
|
||||
}
|
||||
@@ -991,10 +994,25 @@ async fn sync_opml(
|
||||
}
|
||||
|
||||
/// Why this enclosure should not be downloaded, if it should not be.
|
||||
fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'static str> {
|
||||
fn reject(
|
||||
cfg: &config::Config,
|
||||
feed_cfg: &config::Feed,
|
||||
entry: &feed::Entry,
|
||||
enc: &feed::Enclosure,
|
||||
) -> Option<&'static str> {
|
||||
let url = enc.url.as_str();
|
||||
if !feed_cfg.auto_download {
|
||||
return Some("auto_download is off");
|
||||
}
|
||||
// Blog feeds put the article's header image in an <enclosure>; without this a text
|
||||
// feed reads as a podcast full of episodes and fills the disk with artwork.
|
||||
let wanted = feed_cfg
|
||||
.media_types
|
||||
.as_deref()
|
||||
.unwrap_or(&cfg.general.media_types);
|
||||
if !config::wanted_media(enc.mime.as_deref(), wanted) {
|
||||
return Some("not a wanted media type");
|
||||
}
|
||||
if entry.explicit && !feed_cfg.allow_explicit {
|
||||
return Some("explicit");
|
||||
}
|
||||
|
||||
13
src/web.rs
13
src/web.rs
@@ -283,7 +283,7 @@ mod tests {
|
||||
|
||||
fn feed(url: &str) -> crate::config::Feed {
|
||||
crate::config::Feed {
|
||||
url: url.into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
auto_download: true, max_new_per_check: None, username: None,
|
||||
password: None, password_env: None,
|
||||
}
|
||||
@@ -754,6 +754,7 @@ async fn import_opml(
|
||||
url,
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
@@ -776,6 +777,7 @@ struct Settings {
|
||||
schedule: String,
|
||||
every_mins: u64,
|
||||
max_new_per_check: usize,
|
||||
media_types: Vec<String>,
|
||||
download_dir: String,
|
||||
max_total_gb: f64,
|
||||
max_age_days: u64,
|
||||
@@ -787,6 +789,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
|
||||
schedule: cfg.general.schedule.clone(),
|
||||
every_mins: cfg.general.interval(),
|
||||
max_new_per_check: cfg.general.max_new_per_check,
|
||||
media_types: cfg.general.media_types.clone(),
|
||||
download_dir: cfg.general.download_dir.display().to_string(),
|
||||
max_total_gb: cfg.general.max_total_gb,
|
||||
max_age_days: cfg.general.max_age_days,
|
||||
@@ -797,6 +800,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
|
||||
struct SettingsPatch {
|
||||
schedule: Option<String>,
|
||||
max_new_per_check: Option<usize>,
|
||||
media_types: Option<Vec<String>>,
|
||||
max_total_gb: Option<f64>,
|
||||
max_age_days: Option<u64>,
|
||||
}
|
||||
@@ -820,6 +824,13 @@ async fn patch_settings(
|
||||
if let Some(v) = body.max_new_per_check {
|
||||
cfg.general.max_new_per_check = v;
|
||||
}
|
||||
if let Some(v) = body.media_types {
|
||||
cfg.general.media_types = v
|
||||
.into_iter()
|
||||
.map(|t| t.trim().to_lowercase())
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect();
|
||||
}
|
||||
if let Some(v) = body.max_total_gb {
|
||||
cfg.general.max_total_gb = v.max(0.0);
|
||||
}
|
||||
|
||||
@@ -606,7 +606,8 @@ function epEl(e){
|
||||
${num?`<span>${num}</span><span class="dot"></span>`:''}
|
||||
<span>${dateOf(e.published)}</span>
|
||||
${left?'<span class="dot"></span><span>'+left+'</span>':''}
|
||||
${enc?`<span class="dot"></span><span class="chip ${esc(enc.state)}">${has?'downloaded':esc(enc.state)}</span>`:''}
|
||||
${enc?`<span class="dot"></span><span class="chip ${esc(enc.state)}">${
|
||||
has?'downloaded':(enc.state==='skipped'?kindOf(enc):esc(enc.state))}</span>`:''}
|
||||
${enc&&enc.length?`<span>${mb(enc.length)}</span>`:''}
|
||||
${e.flagged?'<span class="dot"></span><span>★ kept</span>':''}
|
||||
</div>
|
||||
@@ -615,7 +616,8 @@ function epEl(e){
|
||||
</div>
|
||||
<div class="rowacts">
|
||||
${has?`<button class="iconbtn" data-a="play" title="Play">▶</button>`:
|
||||
(enc?`<button class="iconbtn" data-a="get" title="Download">⤓</button>`:'')}
|
||||
(enc?`<button class="iconbtn" data-a="get" title="Download this ${
|
||||
enc.state==='skipped'?kindOf(enc):'file'}">⤓</button>`:'')}
|
||||
<button class="iconbtn" data-a="flag" title="${e.flagged?'Stop keeping':'Keep (never auto-delete)'}">${e.flagged?'★':'☆'}</button>
|
||||
<button class="iconbtn" data-a="read" title="Mark ${e.read?'unread':'read'}">${e.read?'○':'●'}</button>
|
||||
${has?`<button class="iconbtn" data-a="del" title="Delete file">🗑</button>`:''}
|
||||
@@ -630,6 +632,18 @@ function epEl(e){
|
||||
if(S.open.has(e.guid)) showNotes(e,el);
|
||||
return el;
|
||||
}
|
||||
/// What an enclosure is, for a row that is not an episode: "image", "pdf", "document".
|
||||
function kindOf(enc){
|
||||
const m=(enc.mime||'').toLowerCase();
|
||||
if(m.startsWith('image/')) return 'image';
|
||||
if(m.startsWith('video/')) return 'video';
|
||||
if(m.startsWith('audio/')) return 'audio';
|
||||
if(m.includes('pdf')) return 'pdf';
|
||||
if(m.includes('torrent')) return 'torrent';
|
||||
const ext=(enc.url||'').split('?')[0].split('.').pop();
|
||||
return (ext && ext.length<=5) ? ext.toLowerCase() : 'file';
|
||||
}
|
||||
|
||||
function feedArt(){ const f=S.feeds.find(x=>x.id===S.feed); return f&&f.image; }
|
||||
|
||||
function toggleNotes(e,el){
|
||||
@@ -937,6 +951,11 @@ async function prefsModal(){
|
||||
<span class="hint">Applies to any feed that does not set its own — including every feed
|
||||
inside an OPML subscription. <b>0 means unlimited</b>, which will pull a whole back
|
||||
catalogue the first time a feed is scanned.</span></div>
|
||||
<div class="field"><label>Download these media types automatically</label>
|
||||
<input type="text" id="gtypes" value="${esc((g.media_types||[]).join(', '))}"
|
||||
placeholder="audio, video">
|
||||
<span class="hint">Anything else is still listed and can be downloaded by hand — blog feeds
|
||||
put article images in enclosures, and those are not episodes. Empty takes everything.</span></div>
|
||||
<div class="field"><label>Disk quota (GB, 0 = unlimited)</label>
|
||||
<input type="number" id="gquota" min="0" step="0.5" value="${g.max_total_gb}">
|
||||
<span class="hint">Over this, the oldest played episodes are deleted first. Starred
|
||||
@@ -952,6 +971,7 @@ async function prefsModal(){
|
||||
await api('/api/settings',{method:'PATCH',body:JSON.stringify({
|
||||
schedule:`every ${Math.max(1,Number($('#gnum').value)||1)}${$('#gunit').value}`,
|
||||
max_new_per_check:Math.max(0,Number($('#gmax').value)||0),
|
||||
media_types:$('#gtypes').value.split(',').map(t=>t.trim()).filter(Boolean),
|
||||
max_total_gb:Number($('#gquota').value)||0,
|
||||
max_age_days:Number($('#gage').value)||0})});
|
||||
closeModal(); toast('Settings saved');
|
||||
|
||||
Reference in New Issue
Block a user