rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subscribe to an OPML, not just import one

Browse Source 
A feed whose body sniffs as OPML is treated as a subscription list and
re-read on every scan, as iPodderX did. Listed feeds become real config
entries grouped under it, inherit its settings, land in one nested folder,
and are scanned in the same run.

When a feed leaves the OPML: removed if nothing was downloaded, kept and
flagged otherwise, so a downloaded file is never orphaned.

folder_for sanitized the whole folder string and would have flattened the
nesting; each segment is sanitized separately now, and a traversal still
cannot escape the download directory. Db::memory() also runs migrate(),
which it did not, so a migration-only column passed tests while missing in
production.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
This commit is contained in:
Ray Slakinski
2026-09-10 15:01:03 +00:00
parent 5f6e2a8dc1
commit c86d698363
9 changed files with 355 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/config.rs
View File

@@ -97,6 +97,10 @@ pub struct Feed {
pub allow_explicit: bool,
#[serde(default = "yes")]
pub auto_download: bool,
/// Set on feeds that came from a subscribed OPML: the id of the OPML feed they
/// belong to. The OPML is re-read on every scan and this list kept in step.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub group: Option<String>,
/// Overrides the global schedule for this feed. Same forms: "every 6h", "2d".
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schedule: Option<String>,
@@ -393,7 +397,7 @@ mod tests {
let mut taken = BTreeMap::new();
taken.insert("the-daily".to_string(), Feed {
url: "u".into(), folder: None, schedule: None, keywords: vec![], allow_explicit: false,
url: "u".into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None,
});
@@ -405,6 +409,7 @@ mod tests {
let mut f = Feed {
url: "https://x/y".into(),
folder: None,
group: None,
schedule: None,
keywords: vec![],
allow_explicit: false,

36
src/db.rs
View File

@@ -21,7 +21,9 @@ CREATE TABLE IF NOT EXISTS feeds (
last_modified TEXT,
last_checked INTEGER,
ttl_mins INTEGER,
last_error TEXT
last_error TEXT,
-- Came from a subscribed OPML that no longer lists it, but has downloads, so kept.
orphaned INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS entries (
@@ -68,6 +70,7 @@ CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid);
fn migrate(conn: &Connection) -> Result<()> {
let wanted: &[(&str, &str, &str)] = &[
("feeds", "image", "TEXT"),
("feeds", "orphaned", "INTEGER NOT NULL DEFAULT 0"),
("entries", "image", "TEXT"),
("entries", "duration", "INTEGER"),
("entries", "episode", "INTEGER"),
@@ -92,6 +95,9 @@ fn migrate(conn: &Connection) -> Result<()> {
pub struct FeedSummary {
pub title: Option<String>,
pub image: Option<String>,
/// Came from a subscribed OPML that no longer lists it, but it has downloads, so it
/// was kept rather than removed.
pub orphaned: bool,
pub last_checked: Option<i64>,
pub last_error: Option<String>,
pub entries: i64,
@@ -119,6 +125,9 @@ impl Db {
pub fn memory() -> Result<Self> {
let conn = Connection::open_in_memory()?;
conn.execute_batch(SCHEMA)?;
// Same path as a real open, so a column added only in migrate() cannot pass the
// tests while being missing in production (or the reverse).
migrate(&conn)?;
Ok(Self { conn: Mutex::new(conn) })
}
@@ -132,7 +141,8 @@ impl Db {
let conn = self.conn.lock().unwrap();
let mut sum: FeedSummary = conn
.query_row(
"SELECT title, image, last_checked, last_error FROM feeds WHERE id = ?1",
"SELECT title, image, last_checked, last_error, coalesce(orphaned, 0)
FROM feeds WHERE id = ?1",
[feed_id],
|r| {
Ok(FeedSummary {
@@ -140,6 +150,7 @@ impl Db {
image: r.get(1)?,
last_checked: r.get(2)?,
last_error: r.get(3)?,
orphaned: r.get::<_, i64>(4)? != 0,
..Default::default()
})
},
@@ -703,6 +714,27 @@ impl Db {
Ok(())
}
/// How many files this feed has on disk. Decides whether a feed dropped from an OPML
/// can be removed or must be kept.
pub fn downloaded_count(&self, feed_id: &str) -> Result<i64> {
let conn = self.conn.lock().unwrap();
Ok(conn.query_row(
"SELECT count(*) FROM enclosures WHERE feed_id = ?1 AND path IS NOT NULL",
[feed_id],
|r| r.get(0),
)?)
}
pub fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> {
let conn = self.conn.lock().unwrap();
conn.execute(
"INSERT INTO feeds (id, url, orphaned) VALUES (?1, '', ?2)
ON CONFLICT(id) DO UPDATE SET orphaned = excluded.orphaned",
rusqlite::params![feed_id, on as i64],
)?;
Ok(())
}
/// Nothing can be in flight the moment the daemon starts, so any row still marked
/// `downloading` is a leftover from a restart or a crash. Left alone it would sit
/// there forever: the pending queue skips it and nothing else ever revisits it.

34
src/download.rs
View File

@@ -268,17 +268,27 @@ fn unique_path(dir: &Path, name: &str) -> PathBuf {
}
/// Download folder for a feed: per-feed name, or per-day when organize = "date".
///
/// A folder may name more than one level ("Subscriptions/Some Show") -- feeds from a
/// subscribed OPML nest under it -- so each segment is sanitized separately rather than
/// letting the sanitizer eat the separator.
pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str>) -> String {
match cfg.general.organize {
Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
Organize::Feed => sanitize(
feed_cfg
Organize::Feed => {
let raw = feed_cfg
.folder
.as_deref()
.or(title)
.filter(|s| !s.trim().is_empty())
.unwrap_or(id),
),
.unwrap_or(id);
raw.split('/')
.map(str::trim)
.filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..")
.map(sanitize)
.collect::<Vec<_>>()
.join("/")
}
}
}
@@ -370,6 +380,22 @@ mod tests {
);
}
#[test]
fn a_folder_can_nest_without_the_sanitizer_eating_the_separator() {
let mut cfg = Config::default();
cfg.general.download_dir = "/tmp".into();
let mut f = crate::config::Feed {
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None,
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
max_new_per_check: None, username: None, password: None, password_env: None,
};
assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");
// A traversal in a folder name must not climb out of the download directory.
f.folder = Some("../../etc/Show".into());
assert_eq!(folder_for(&cfg, "id", &f, None), "etc/Show");
}
#[test]
fn keyword_matching_is_or_across_keywords_and_and_within_one() {
let kws = vec!["deep dive".to_string(), "interview".to_string()];

51
src/feed.rs
View File

@@ -87,6 +87,36 @@ pub async fn fetch(
Ok(Fetched::Body { bytes, etag, last_modified })
}
/// True when a body is an OPML document rather than a feed.
///
/// The original matched on the URL ending in ".opml" (iPXClass.py:34), which misses an
/// OPML served from a URL without that extension. Sniffing the body catches both.
pub fn is_opml(bytes: &[u8]) -> bool {
let head = &bytes[..bytes.len().min(1024)];
let text = String::from_utf8_lossy(head).to_lowercase();
text.contains("<opml")
}
/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text)
.map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
let mut out = vec![];
crate::collect_outlines(&doc.body.outlines, &mut out);
Ok(out)
}
/// The <head><title> of an OPML document.
pub fn opml_title(bytes: &[u8]) -> Option<String> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text).ok()?;
doc.head
.and_then(|h| h.title)
.map(|t| t.trim().to_owned())
.filter(|t| !t.is_empty())
}
/// RSS first, then Atom -- the same split the original made on `parsedFeed.version`.
pub fn parse(bytes: &[u8]) -> Result<ParsedFeed> {
match rss::Channel::read_from(bytes) {
@@ -391,6 +421,27 @@ mod tests {
assert_eq!((p1.season, p1.episode), (Some(8), Some(1)));
}
#[test]
fn opml_is_recognised_and_its_feeds_listed() {
let xml = br#"<opml version="2.0"><head><title>My Subscriptions</title></head><body>
<outline text="Folder">
<outline type="rss" text="Alpha" xmlUrl="https://a.example/rss"/>
<outline type="rss" text="Beta" xmlUrl="https://b.example/rss"/>
</outline>
<outline text="Not a feed"/>
</body></opml>"#;
assert!(is_opml(xml));
assert_eq!(opml_title(xml).as_deref(), Some("My Subscriptions"));
let feeds = parse_opml(xml).unwrap();
assert_eq!(feeds.len(), 2, "nested folders are walked, non-feed outlines skipped");
assert_eq!(feeds[0], ("Alpha".into(), "https://a.example/rss".into()));
// A feed must never be mistaken for a subscription list.
assert!(!is_opml(include_bytes!("../tests/data/rss2.xml")));
assert!(!is_opml(include_bytes!("../tests/data/atom.xml")));
}
#[test]
fn durations_parse_from_seconds_or_a_clock() {
assert_eq!(parse_duration("5649"), Some(5649));

155
src/main.rs
View File

@@ -89,6 +89,8 @@ pub struct Ctx {
pub torrents: tokio::sync::OnceCell<torrent::Torrents>,
/// Caps how many torrents run at once when they are detached.
pub torrent_slots: Arc<tokio::sync::Semaphore>,
/// Needed because a subscribed OPML rewrites the feed list as it syncs.
pub config_path: PathBuf,
/// Run torrents off the command worker. A torrent takes minutes to fetch metadata and
/// then seeds for up to an hour, and the worker is sequential -- inline, one torrent
/// stops every feed scan, every HTTP download and every status command behind it.
@@ -170,6 +172,7 @@ async fn main() -> Result<()> {
out: if is_daemon { Emitter::socket(events.clone(), false) } else { Emitter::terminal() },
torrents: tokio::sync::OnceCell::new(),
torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),
config_path: config_path.clone(),
detach_torrents: is_daemon,
});
@@ -382,6 +385,7 @@ pub async fn add_one(
let probe = config::Feed {
url: url.to_owned(),
folder: folder.clone(),
group: None,
schedule: None,
keywords: keywords.clone(),
allow_explicit: false,
@@ -393,6 +397,11 @@ pub async fn add_one(
};
let title = match feed::fetch(&ctx.client, &probe, None, None).await {
// An OPML subscription is named from its own <head><title>, not by trying to
// parse it as a feed and falling back to the hostname.
Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
feed::opml_title(&bytes).unwrap_or_else(|| url_stem(url))
}
Ok(feed::Fetched::Body { bytes, .. }) => feed::parse(&bytes)
.ok()
.and_then(|f| f.title)
@@ -448,6 +457,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path
config::Feed {
url,
folder: None,
group: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
@@ -546,6 +556,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
}
let mut scanned = 0;
let mut fresh: Vec<String> = vec![];
for (id, feed_cfg) in cfg
.feeds
.iter()
@@ -567,17 +578,29 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
match scan_one(ctx, id, feed_cfg, &state).await {
Ok(Some(s)) => ctx.out.emit(Event::FeedDone {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
downloaded: s.downloaded,
failed: s.failed,
torrents: s.torrents,
}),
Ok(None) => ctx.out.emit(Event::FeedSkip {
Ok(Outcome::NotModified) => ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: "not modified".into(),
}),
Ok(Outcome::Opml { added, removed, kept, total }) => {
ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: format!(
"OPML: {total} feed(s) listed, {} added, {removed} unsubscribed, {kept} kept without a listing",
added.len()
),
});
// Read them in the same pass, as the original did, rather than making
// the user wait a whole interval for a newly listed show.
fresh.extend(added);
}
Err(e) => {
// One bad feed must not end the scan.
let msg = format!("{e:#}");
@@ -586,6 +609,31 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
}
}
}
// Feeds a subscribed OPML just introduced: scan them now, in this run.
if !fresh.is_empty() {
let cfg = ctx.cfg();
for id in &fresh {
let Some(feed_cfg) = cfg.feeds.get(id) else { continue };
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
let state = ctx.db.http_state(id)?;
match scan_one(ctx, id, feed_cfg, &state).await {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
downloaded: s.downloaded,
failed: s.failed,
torrents: s.torrents,
}),
Ok(_) => {}
Err(e) => {
let msg = format!("{e:#}");
ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() });
ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?;
}
}
}
}
ctx.out.emit(Event::ScanDone { feeds: scanned });
Ok(())
}
@@ -611,13 +659,20 @@ struct Scan {
torrents: usize,
}
/// Ok(None) means 304.
/// What a scan of one feed turned out to be.
enum Outcome {
NotModified,
Feed(Scan),
/// The URL served an OPML document, so it is a subscription list rather than a feed.
Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
}
async fn scan_one(
ctx: &Arc<Ctx>,
id: &str,
feed_cfg: &config::Feed,
state: &db::HttpState,
) -> Result<Option<Scan>> {
) -> Result<Outcome> {
let fetched = feed::fetch(
&ctx.client,
feed_cfg,
@@ -629,11 +684,18 @@ async fn scan_one(
let (bytes, etag, last_modified) = match fetched {
feed::Fetched::NotModified => {
ctx.db.touch_feed(id, &feed_cfg.url)?;
return Ok(None);
return Ok(Outcome::NotModified);
}
feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified),
};
// A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml"
// URL; sniffing the body also catches one served from a URL without that extension.
if feed::is_opml(&bytes) {
ctx.db.touch_feed(id, &feed_cfg.url)?;
return sync_opml(ctx, id, feed_cfg, &bytes).await;
}
let parsed = feed::parse(&bytes)?;
ctx.db.record_feed(
id,
@@ -738,7 +800,88 @@ async fn scan_one(
}
}
Ok(Some(scan))
Ok(Outcome::Feed(scan))
}
/// Brings the feed list in step with a subscribed OPML.
///
/// New entries are added under the OPML's group and folder. An entry that has gone from
/// the OPML is unsubscribed *only if nothing was ever downloaded for it* -- otherwise it
/// is kept and flagged, because dropping it would orphan files on disk with nothing in
/// the UI to explain them.
async fn sync_opml(
ctx: &Arc<Ctx>,
parent_id: &str,
parent: &config::Feed,
bytes: &[u8],
) -> Result<Outcome> {
let listed = feed::parse_opml(bytes)?;
let mut cfg = (*ctx.cfg()).clone();
let base_folder = parent
.folder
.clone()
.or_else(|| ctx.db.feed_summary(parent_id).ok().and_then(|s| s.title))
.unwrap_or_else(|| parent_id.to_owned());
let mut added = vec![];
for (title, url) in &listed {
if let Some((id, _)) = cfg.feeds.iter().find(|(_, f)| &f.url == url) {
// Already subscribed. If it had been flagged as gone, it is back.
let id = id.clone();
let _ = ctx.db.set_orphaned(&id, false);
continue;
}
let id = config::unique_slug(title, &cfg.feeds);
cfg.feeds.insert(
id.clone(),
config::Feed {
url: url.clone(),
// Nested, so the whole subscription lands in one folder.
folder: Some(format!("{base_folder}/{title}")),
group: Some(parent_id.to_owned()),
schedule: parent.schedule.clone(),
keywords: parent.keywords.clone(),
allow_explicit: parent.allow_explicit,
auto_download: parent.auto_download,
max_new_per_check: parent.max_new_per_check,
username: parent.username.clone(),
password: parent.password.clone(),
password_env: parent.password_env.clone(),
},
);
added.push(id);
}
// Anything in this group the OPML no longer lists.
let gone: Vec<String> = cfg
.feeds
.iter()
.filter(|(_, f)| f.group.as_deref() == Some(parent_id))
.filter(|(_, f)| !listed.iter().any(|(_, u)| u == &f.url))
.map(|(id, _)| id.clone())
.collect();
let mut removed = 0;
let mut kept = 0;
for id in gone {
if ctx.db.downloaded_count(&id).unwrap_or(1) > 0 {
// Never orphan a downloaded file: keep the feed and say why in the UI.
ctx.db.set_orphaned(&id, true)?;
kept += 1;
tracing::info!(feed = %id, "dropped from the OPML but has downloads; keeping it");
} else {
cfg.feeds.remove(&id);
removed += 1;
tracing::info!(feed = %id, "dropped from the OPML with nothing downloaded; unsubscribed");
}
}
if !added.is_empty() || removed > 0 {
cfg.save(&ctx.config_path)?;
ctx.reload_cfg(&ctx.config_path)?;
}
Ok(Outcome::Opml { added, removed, kept, total: listed.len() })
}
/// Why this enclosure should not be downloaded, if it should not be.

9
src/web.rs
View File

@@ -155,6 +155,10 @@ struct FeedRow {
allow_explicit: bool,
auto_download: bool,
max_new_per_check: Option<usize>,
/// The OPML subscription this feed came from, if any.
group: Option<String>,
/// In a group, but the OPML no longer lists it. Kept because it has downloads.
orphaned: bool,
schedule: Option<String>,
/// The feed's own override in minutes, so the UI need not re-parse the string.
schedule_mins: Option<u64>,
@@ -184,6 +188,8 @@ async fn feeds(State(state): State<WebState>) -> Result<Json<Vec<FeedRow>>, ApiE
allow_explicit: feed.allow_explicit,
auto_download: feed.auto_download,
max_new_per_check: feed.max_new_per_check,
group: feed.group.clone(),
orphaned: s.orphaned,
schedule: feed.schedule.clone(),
schedule_mins: feed
.schedule
@@ -271,7 +277,7 @@ mod tests {
fn feed(url: &str) -> crate::config::Feed {
crate::config::Feed {
url: url.into(), folder: None, schedule: None, keywords: vec![], allow_explicit: false,
url: url.into(), folder: None, group: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None,
}
@@ -722,6 +728,7 @@ async fn import_opml(
crate::config::Feed {
url,
folder: None,
group: None,
schedule: None,
keywords: vec![],
allow_explicit: false,