rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subscribe to an OPML, not just import one

Browse Source 
A feed whose body sniffs as OPML is treated as a subscription list and
re-read on every scan, as iPodderX did. Listed feeds become real config
entries grouped under it, inherit its settings, land in one nested folder,
and are scanned in the same run.

When a feed leaves the OPML: removed if nothing was downloaded, kept and
flagged otherwise, so a downloaded file is never orphaned.

folder_for sanitized the whole folder string and would have flattened the
nesting; each segment is sanitized separately now, and a traversal still
cannot escape the download directory. Db::memory() also runs migrate(),
which it did not, so a migration-only column passed tests while missing in
production.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
This commit is contained in:
Ray Slakinski
2026-09-10 15:01:03 +00:00
parent 5f6e2a8dc1
commit c86d698363
9 changed files with 355 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
src/main.rs
View File

@@ -89,6 +89,8 @@ pub struct Ctx {
pub torrents: tokio::sync::OnceCell<torrent::Torrents>,
/// Caps how many torrents run at once when they are detached.
pub torrent_slots: Arc<tokio::sync::Semaphore>,
/// Needed because a subscribed OPML rewrites the feed list as it syncs.
pub config_path: PathBuf,
/// Run torrents off the command worker. A torrent takes minutes to fetch metadata and
/// then seeds for up to an hour, and the worker is sequential -- inline, one torrent
/// stops every feed scan, every HTTP download and every status command behind it.
@@ -170,6 +172,7 @@ async fn main() -> Result<()> {
out: if is_daemon { Emitter::socket(events.clone(), false) } else { Emitter::terminal() },
torrents: tokio::sync::OnceCell::new(),
torrent_slots: Arc::new(tokio::sync::Semaphore::new(2)),
config_path: config_path.clone(),
detach_torrents: is_daemon,
});
@@ -382,6 +385,7 @@ pub async fn add_one(
let probe = config::Feed {
url: url.to_owned(),
folder: folder.clone(),
group: None,
schedule: None,
keywords: keywords.clone(),
allow_explicit: false,
@@ -393,6 +397,11 @@ pub async fn add_one(
};
let title = match feed::fetch(&ctx.client, &probe, None, None).await {
// An OPML subscription is named from its own <head><title>, not by trying to
// parse it as a feed and falling back to the hostname.
Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
feed::opml_title(&bytes).unwrap_or_else(|| url_stem(url))
}
Ok(feed::Fetched::Body { bytes, .. }) => feed::parse(&bytes)
.ok()
.and_then(|f| f.title)
@@ -448,6 +457,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path
config::Feed {
url,
folder: None,
group: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
@@ -546,6 +556,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
}
let mut scanned = 0;
let mut fresh: Vec<String> = vec![];
for (id, feed_cfg) in cfg
.feeds
.iter()
@@ -567,17 +578,29 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
match scan_one(ctx, id, feed_cfg, &state).await {
Ok(Some(s)) => ctx.out.emit(Event::FeedDone {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
downloaded: s.downloaded,
failed: s.failed,
torrents: s.torrents,
}),
Ok(None) => ctx.out.emit(Event::FeedSkip {
Ok(Outcome::NotModified) => ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: "not modified".into(),
}),
Ok(Outcome::Opml { added, removed, kept, total }) => {
ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: format!(
"OPML: {total} feed(s) listed, {} added, {removed} unsubscribed, {kept} kept without a listing",
added.len()
),
});
// Read them in the same pass, as the original did, rather than making
// the user wait a whole interval for a newly listed show.
fresh.extend(added);
}
Err(e) => {
// One bad feed must not end the scan.
let msg = format!("{e:#}");
@@ -586,6 +609,31 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
}
}
}
// Feeds a subscribed OPML just introduced: scan them now, in this run.
if !fresh.is_empty() {
let cfg = ctx.cfg();
for id in &fresh {
let Some(feed_cfg) = cfg.feeds.get(id) else { continue };
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
let state = ctx.db.http_state(id)?;
match scan_one(ctx, id, feed_cfg, &state).await {
Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone {
feed: id.clone(),
new: s.new_entries,
downloaded: s.downloaded,
failed: s.failed,
torrents: s.torrents,
}),
Ok(_) => {}
Err(e) => {
let msg = format!("{e:#}");
ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() });
ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?;
}
}
}
}
ctx.out.emit(Event::ScanDone { feeds: scanned });
Ok(())
}
@@ -611,13 +659,20 @@ struct Scan {
torrents: usize,
}
/// Ok(None) means 304.
/// What a scan of one feed turned out to be.
enum Outcome {
NotModified,
Feed(Scan),
/// The URL served an OPML document, so it is a subscription list rather than a feed.
Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
}
async fn scan_one(
ctx: &Arc<Ctx>,
id: &str,
feed_cfg: &config::Feed,
state: &db::HttpState,
) -> Result<Option<Scan>> {
) -> Result<Outcome> {
let fetched = feed::fetch(
&ctx.client,
feed_cfg,
@@ -629,11 +684,18 @@ async fn scan_one(
let (bytes, etag, last_modified) = match fetched {
feed::Fetched::NotModified => {
ctx.db.touch_feed(id, &feed_cfg.url)?;
return Ok(None);
return Ok(Outcome::NotModified);
}
feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified),
};
// A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml"
// URL; sniffing the body also catches one served from a URL without that extension.
if feed::is_opml(&bytes) {
ctx.db.touch_feed(id, &feed_cfg.url)?;
return sync_opml(ctx, id, feed_cfg, &bytes).await;
}
let parsed = feed::parse(&bytes)?;
ctx.db.record_feed(
id,
@@ -738,7 +800,88 @@ async fn scan_one(
}
}
Ok(Some(scan))
Ok(Outcome::Feed(scan))
}
/// Brings the feed list in step with a subscribed OPML.
///
/// New entries are added under the OPML's group and folder. An entry that has gone from
/// the OPML is unsubscribed *only if nothing was ever downloaded for it* -- otherwise it
/// is kept and flagged, because dropping it would orphan files on disk with nothing in
/// the UI to explain them.
async fn sync_opml(
ctx: &Arc<Ctx>,
parent_id: &str,
parent: &config::Feed,
bytes: &[u8],
) -> Result<Outcome> {
let listed = feed::parse_opml(bytes)?;
let mut cfg = (*ctx.cfg()).clone();
let base_folder = parent
.folder
.clone()
.or_else(|| ctx.db.feed_summary(parent_id).ok().and_then(|s| s.title))
.unwrap_or_else(|| parent_id.to_owned());
let mut added = vec![];
for (title, url) in &listed {
if let Some((id, _)) = cfg.feeds.iter().find(|(_, f)| &f.url == url) {
// Already subscribed. If it had been flagged as gone, it is back.
let id = id.clone();
let _ = ctx.db.set_orphaned(&id, false);
continue;
}
let id = config::unique_slug(title, &cfg.feeds);
cfg.feeds.insert(
id.clone(),
config::Feed {
url: url.clone(),
// Nested, so the whole subscription lands in one folder.
folder: Some(format!("{base_folder}/{title}")),
group: Some(parent_id.to_owned()),
schedule: parent.schedule.clone(),
keywords: parent.keywords.clone(),
allow_explicit: parent.allow_explicit,
auto_download: parent.auto_download,
max_new_per_check: parent.max_new_per_check,
username: parent.username.clone(),
password: parent.password.clone(),
password_env: parent.password_env.clone(),
},
);
added.push(id);
}
// Anything in this group the OPML no longer lists.
let gone: Vec<String> = cfg
.feeds
.iter()
.filter(|(_, f)| f.group.as_deref() == Some(parent_id))
.filter(|(_, f)| !listed.iter().any(|(_, u)| u == &f.url))
.map(|(id, _)| id.clone())
.collect();
let mut removed = 0;
let mut kept = 0;
for id in gone {
if ctx.db.downloaded_count(&id).unwrap_or(1) > 0 {
// Never orphan a downloaded file: keep the feed and say why in the UI.
ctx.db.set_orphaned(&id, true)?;
kept += 1;
tracing::info!(feed = %id, "dropped from the OPML but has downloads; keeping it");
} else {
cfg.feeds.remove(&id);
removed += 1;
tracing::info!(feed = %id, "dropped from the OPML with nothing downloaded; unsubscribed");
}
}
if !added.is_empty() || removed > 0 {
cfg.save(&ctx.config_path)?;
ctx.reload_cfg(&ctx.config_path)?;
}
Ok(Outcome::Opml { added, removed, kept, total: listed.len() })
}
/// Why this enclosure should not be downloaded, if it should not be.