Do not auto-download enclosures that are not audio or video
Blog feeds put each article's header image in an <enclosure>, so a text feed read as a podcast full of episodes: 149 images, 74 MB across 11 feeds. media_types defaults to audio and video, with a per-feed override. Such enclosures stay listed and stay downloadable by hand; the row names what it is rather than saying "skipped". An unknown type is allowed, since the real type is only known after downloading, and a torrent is allowed as a container judged once unpacked. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
This commit is contained in:
@@ -38,6 +38,10 @@ pub struct General {
|
||||
/// Unlimited by default was a trap: subscribing to an OPML of 80 feeds then pulled
|
||||
/// every back-catalogue episode at once. 0 means unlimited, deliberately chosen.
|
||||
pub max_new_per_check: usize,
|
||||
/// Top-level media types worth downloading. Blog feeds put each article's header
|
||||
/// image in an <enclosure>, so taking everything filled the disk with artwork and
|
||||
/// counted it as episodes. Empty means take anything.
|
||||
pub media_types: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
|
||||
@@ -108,6 +112,9 @@ pub struct Feed {
|
||||
/// Overrides the global schedule for this feed. Same forms: "every 6h", "2d".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub schedule: Option<String>,
|
||||
/// Media types for this feed. None follows `[general]`; an empty list takes anything.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub media_types: Option<Vec<String>>,
|
||||
/// Cap on new downloads per scan for this feed. None follows `[general]`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_new_per_check: Option<usize>,
|
||||
@@ -135,6 +142,7 @@ impl Default for General {
|
||||
max_total_gb: 0.0,
|
||||
max_age_days: 0,
|
||||
max_new_per_check: 3,
|
||||
media_types: vec!["audio".into(), "video".into()],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -276,6 +284,25 @@ fn default_socket() -> PathBuf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an enclosure's type is one we want.
|
||||
///
|
||||
/// An unknown type is allowed: the real type is only known after downloading, and
|
||||
/// refusing everything untyped would drop feeds that simply omit the attribute.
|
||||
pub fn wanted_media(mime: Option<&str>, wanted: &[String]) -> bool {
|
||||
if wanted.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let Some(mime) = mime.map(str::trim).filter(|m| !m.is_empty()) else {
|
||||
return true;
|
||||
};
|
||||
let top = mime.split('/').next().unwrap_or(mime).to_ascii_lowercase();
|
||||
// A .torrent is a container for media, not media itself; judge it once unpacked.
|
||||
if mime.to_ascii_lowercase().contains("torrent") {
|
||||
return true;
|
||||
}
|
||||
wanted.iter().any(|w| w.trim().eq_ignore_ascii_case(&top) || w.trim().eq_ignore_ascii_case(mime))
|
||||
}
|
||||
|
||||
/// Feed ids are the TOML table key, so they must be readable and punctuation-free.
|
||||
pub fn slug(text: &str) -> String {
|
||||
let mut out = String::new();
|
||||
@@ -391,6 +418,26 @@ mod tests {
|
||||
assert_eq!(t.ports(), (6881, 6889), "reversed range is not a range");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_types_keep_article_artwork_out() {
|
||||
let want = vec!["audio".to_string(), "video".to_string()];
|
||||
assert!(wanted_media(Some("audio/mpeg"), &want));
|
||||
assert!(wanted_media(Some("audio/mp4"), &want));
|
||||
assert!(wanted_media(Some("video/quicktime"), &want));
|
||||
assert!(!wanted_media(Some("image/jpeg"), &want), "a blog header image is not an episode");
|
||||
assert!(!wanted_media(Some("text/html"), &want));
|
||||
|
||||
// A torrent is a container; what is inside is judged after unpacking.
|
||||
assert!(wanted_media(Some("application/x-bittorrent"), &want));
|
||||
// Unknown type: only discoverable by downloading, so do not refuse it outright.
|
||||
assert!(wanted_media(None, &want));
|
||||
assert!(wanted_media(Some(""), &want));
|
||||
// An empty list means take anything, which is how it behaved before.
|
||||
assert!(wanted_media(Some("image/jpeg"), &[]));
|
||||
// A full type can be named exactly.
|
||||
assert!(wanted_media(Some("image/jpeg"), &["image/jpeg".to_string()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slugs_are_readable_and_unique() {
|
||||
assert_eq!(slug("Accidental Tech Podcast"), "accidental-tech-podcast");
|
||||
@@ -402,7 +449,7 @@ mod tests {
|
||||
|
||||
let mut taken = BTreeMap::new();
|
||||
taken.insert("the-daily".to_string(), Feed {
|
||||
url: "u".into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
auto_download: true, max_new_per_check: None, username: None,
|
||||
password: None, password_env: None,
|
||||
});
|
||||
@@ -415,6 +462,7 @@ mod tests {
|
||||
url: "https://x/y".into(),
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
|
||||
@@ -385,7 +385,7 @@ mod tests {
|
||||
let mut cfg = Config::default();
|
||||
cfg.general.download_dir = "/tmp".into();
|
||||
let mut f = crate::config::Feed {
|
||||
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None,
|
||||
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
|
||||
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
|
||||
max_new_per_check: None, username: None, password: None, password_env: None,
|
||||
};
|
||||
|
||||
22
src/main.rs
22
src/main.rs
@@ -412,6 +412,7 @@ pub async fn add_one(
|
||||
url: url.to_owned(),
|
||||
folder: folder.clone(),
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: keywords.clone(),
|
||||
allow_explicit: false,
|
||||
@@ -494,6 +495,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path
|
||||
url,
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
@@ -711,6 +713,7 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
|
||||
url: m.url.clone(),
|
||||
folder: Some(format!("{base}/{title}")),
|
||||
group: Some(m.group_id.clone()),
|
||||
media_types: parent.and_then(|p| p.media_types.clone()),
|
||||
schedule: parent.and_then(|p| p.schedule.clone()),
|
||||
keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(),
|
||||
allow_explicit: parent.is_some_and(|p| p.allow_explicit),
|
||||
@@ -837,7 +840,7 @@ async fn scan_one(
|
||||
}
|
||||
// Filters run once, at discovery, and are recorded in `state`. The download
|
||||
// queue below is then just "everything still pending".
|
||||
if let Some(reason) = reject(feed_cfg, entry, &enc.url) {
|
||||
if let Some(reason) = reject(&ctx.cfg(), feed_cfg, entry, enc) {
|
||||
ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
|
||||
}
|
||||
}
|
||||
@@ -991,10 +994,25 @@ async fn sync_opml(
|
||||
}
|
||||
|
||||
/// Why this enclosure should not be downloaded, if it should not be.
|
||||
fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'static str> {
|
||||
fn reject(
|
||||
cfg: &config::Config,
|
||||
feed_cfg: &config::Feed,
|
||||
entry: &feed::Entry,
|
||||
enc: &feed::Enclosure,
|
||||
) -> Option<&'static str> {
|
||||
let url = enc.url.as_str();
|
||||
if !feed_cfg.auto_download {
|
||||
return Some("auto_download is off");
|
||||
}
|
||||
// Blog feeds put the article's header image in an <enclosure>; without this a text
|
||||
// feed reads as a podcast full of episodes and fills the disk with artwork.
|
||||
let wanted = feed_cfg
|
||||
.media_types
|
||||
.as_deref()
|
||||
.unwrap_or(&cfg.general.media_types);
|
||||
if !config::wanted_media(enc.mime.as_deref(), wanted) {
|
||||
return Some("not a wanted media type");
|
||||
}
|
||||
if entry.explicit && !feed_cfg.allow_explicit {
|
||||
return Some("explicit");
|
||||
}
|
||||
|
||||
13
src/web.rs
13
src/web.rs
@@ -283,7 +283,7 @@ mod tests {
|
||||
|
||||
fn feed(url: &str) -> crate::config::Feed {
|
||||
crate::config::Feed {
|
||||
url: url.into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
|
||||
auto_download: true, max_new_per_check: None, username: None,
|
||||
password: None, password_env: None,
|
||||
}
|
||||
@@ -754,6 +754,7 @@ async fn import_opml(
|
||||
url,
|
||||
folder: None,
|
||||
group: None,
|
||||
media_types: None,
|
||||
schedule: None,
|
||||
keywords: vec![],
|
||||
allow_explicit: false,
|
||||
@@ -776,6 +777,7 @@ struct Settings {
|
||||
schedule: String,
|
||||
every_mins: u64,
|
||||
max_new_per_check: usize,
|
||||
media_types: Vec<String>,
|
||||
download_dir: String,
|
||||
max_total_gb: f64,
|
||||
max_age_days: u64,
|
||||
@@ -787,6 +789,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
|
||||
schedule: cfg.general.schedule.clone(),
|
||||
every_mins: cfg.general.interval(),
|
||||
max_new_per_check: cfg.general.max_new_per_check,
|
||||
media_types: cfg.general.media_types.clone(),
|
||||
download_dir: cfg.general.download_dir.display().to_string(),
|
||||
max_total_gb: cfg.general.max_total_gb,
|
||||
max_age_days: cfg.general.max_age_days,
|
||||
@@ -797,6 +800,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> {
|
||||
struct SettingsPatch {
|
||||
schedule: Option<String>,
|
||||
max_new_per_check: Option<usize>,
|
||||
media_types: Option<Vec<String>>,
|
||||
max_total_gb: Option<f64>,
|
||||
max_age_days: Option<u64>,
|
||||
}
|
||||
@@ -820,6 +824,13 @@ async fn patch_settings(
|
||||
if let Some(v) = body.max_new_per_check {
|
||||
cfg.general.max_new_per_check = v;
|
||||
}
|
||||
if let Some(v) = body.media_types {
|
||||
cfg.general.media_types = v
|
||||
.into_iter()
|
||||
.map(|t| t.trim().to_lowercase())
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect();
|
||||
}
|
||||
if let Some(v) = body.max_total_gb {
|
||||
cfg.general.max_total_gb = v.max(0.0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user