rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep OPML feeds out of config, cap downloads, log daemon work

Browse Source 
Writing 82 derived feeds into a hand-edited config.toml made it
unreadable. The OPML is the source of truth, so its feeds are re-derived
each scan and held in the database, inheriting the subscription's
settings; editing one promotes it to a real entry. A migration moves
existing children out -- 611 lines to 38 -- keeping all entries and files.

max_new_per_check defaulted to unlimited, so subscribing to an OPML of 82
feeds pulled whole back catalogues. It now defaults to 3 via [general],
capping every feed that does not set its own, and the pending queue orders
by publish date so a cap of 3 means the three newest.

Scans and downloads travelled as socket events only, so the log view
showed no daemon activity. They are mirrored into tracing, with routine
skips at debug -- at 82 feeds those alone would flush the buffer.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AdXho5tTkjFLeUXKbEjKBh
This commit is contained in:
Ray Slakinski
2026-09-10 15:15:54 +00:00
parent c86d698363
commit 665d5b8ecb
9 changed files with 456 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
src/main.rs
View File

@@ -217,6 +217,12 @@ async fn daemon(
anyhow::bail!("a daemon is already listening on {}", socket.display());
}
match migrate_opml_children(&ctx) {
Ok(n) if n > 0 => tracing::info!(count = n, "moved OPML feeds out of config.toml into the database"),
Ok(_) => {}
Err(e) => tracing::warn!(error = ?e, "could not tidy OPML feeds out of the config"),
}
match ctx.db.requeue_interrupted() {
Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"),
Ok(_) => {}
@@ -549,19 +555,17 @@ fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
let cfg = ctx.cfg();
let subs = subscriptions(ctx)?;
if let Some(id) = only
&& !cfg.feeds.contains_key(id)
&& !subs.iter().any(|s| s.id == id)
{
anyhow::bail!("no feed with id {id:?}");
}
let mut scanned = 0;
let mut fresh: Vec<String> = vec![];
for (id, feed_cfg) in cfg
.feeds
.iter()
.filter(|(id, _)| only.is_none_or(|o| o == *id))
{
for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) {
let (id, feed_cfg) = (&sub.id, &sub.cfg);
let state = ctx.db.http_state(id)?;
if !force && let Some(last) = state.last_checked {
@@ -611,9 +615,11 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
}
// Feeds a subscribed OPML just introduced: scan them now, in this run.
if !fresh.is_empty() {
let cfg = ctx.cfg();
let subs = subscriptions(ctx)?;
for id in &fresh {
let Some(feed_cfg) = cfg.feeds.get(id) else { continue };
let Some(feed_cfg) = subs.iter().find(|s| &s.id == id).map(|s| &s.cfg) else {
continue;
};
scanned += 1;
ctx.out.emit(Event::FeedStart { feed: id.clone() });
let state = ctx.db.http_state(id)?;
@@ -638,6 +644,89 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> {
Ok(())
}
/// One feed to scan: either an entry you wrote in config.toml, or one derived from an
/// OPML subscription and held only in the database.
pub struct Sub {
pub id: String,
pub cfg: config::Feed,
/// True when it came from an OPML and has no config entry of its own.
pub managed: bool,
}
/// Everything to scan: your config entries, plus whatever the OPML subscriptions listed.
///
/// A derived feed borrows its parent's settings wholesale. That is why it needs no config
/// entry -- there is nothing to store but its URL and where it came from.
pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
let cfg = ctx.cfg();
let mut out: Vec<Sub> = cfg
.feeds
.iter()
.map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false })
.collect();
for m in ctx.db.managed_feeds()? {
if cfg.feeds.contains_key(&m.id) {
continue; // promoted to config at some point; that entry wins
}
let parent = cfg.feeds.get(&m.group_id);
let base = parent
.and_then(|p| p.folder.clone())
.or_else(|| ctx.db.feed_summary(&m.group_id).ok().and_then(|s| s.title))
.unwrap_or_else(|| m.group_id.clone());
let title = m.title.clone().unwrap_or_else(|| m.id.clone());
out.push(Sub {
id: m.id.clone(),
cfg: config::Feed {
url: m.url.clone(),
folder: Some(format!("{base}/{title}")),
group: Some(m.group_id.clone()),
schedule: parent.and_then(|p| p.schedule.clone()),
keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(),
allow_explicit: parent.is_some_and(|p| p.allow_explicit),
auto_download: parent.is_none_or(|p| p.auto_download),
max_new_per_check: parent.and_then(|p| p.max_new_per_check),
username: parent.and_then(|p| p.username.clone()),
password: parent.and_then(|p| p.password.clone()),
password_env: parent.and_then(|p| p.password_env.clone()),
},
managed: true,
});
}
out.sort_by(|a, b| a.id.cmp(&b.id));
Ok(out)
}
/// Moves OPML children that older versions wrote into config.toml over to the database.
/// They were never yours to edit, and 80-odd of them made the file unreadable.
fn migrate_opml_children(ctx: &Ctx) -> Result<usize> {
let cfg = (*ctx.cfg()).clone();
let children: Vec<(String, config::Feed)> = cfg
.feeds
.iter()
.filter(|(_, f)| f.group.is_some())
.map(|(id, f)| (id.clone(), f.clone()))
.collect();
if children.is_empty() {
return Ok(0);
}
let mut fresh = cfg.clone();
for (id, f) in &children {
let group = f.group.clone().unwrap_or_default();
let title = ctx
.db
.feed_summary(id)
.ok()
.and_then(|s| s.title)
.unwrap_or_else(|| id.clone());
ctx.db.upsert_managed(id, &f.url, &title, &group)?;
fresh.feeds.remove(id);
}
fresh.save(&ctx.config_path)?;
ctx.reload_cfg(&ctx.config_path)?;
Ok(children.len())
}
/// Seconds to wait before re-checking a feed.
///
/// A per-feed schedule is an explicit instruction and wins outright. Without one, the
@@ -724,7 +813,11 @@ async fn scan_one(
}
}
let budget = feed_cfg.max_new_per_check.unwrap_or(usize::MAX);
// An unset per-feed cap follows the global one; 0 there means unlimited.
let budget = feed_cfg.max_new_per_check.unwrap_or_else(|| {
let g = ctx.cfg().general.max_new_per_check;
if g == 0 { usize::MAX } else { g }
});
if feed_cfg.auto_download && budget > 0 {
let cfg = ctx.cfg();
let folder = download::folder_for(&cfg, id, feed_cfg, parsed.title.as_deref());
@@ -816,71 +909,54 @@ async fn sync_opml(
bytes: &[u8],
) -> Result<Outcome> {
let listed = feed::parse_opml(bytes)?;
let mut cfg = (*ctx.cfg()).clone();
let base_folder = parent
.folder
.clone()
.or_else(|| ctx.db.feed_summary(parent_id).ok().and_then(|s| s.title))
.unwrap_or_else(|| parent_id.to_owned());
if let Some(title) = feed::opml_title(bytes) {
ctx.db.set_title(parent_id, &title)?;
}
let cfg = ctx.cfg();
let existing = ctx.db.managed_feeds()?;
let mut added = vec![];
for (title, url) in &listed {
if let Some((id, _)) = cfg.feeds.iter().find(|(_, f)| &f.url == url) {
// Already subscribed. If it had been flagged as gone, it is back.
let id = id.clone();
let _ = ctx.db.set_orphaned(&id, false);
// Already known, whether derived or promoted into the config.
if let Some(m) = existing.iter().find(|m| &m.url == url) {
ctx.db.upsert_managed(&m.id, url, title, parent_id)?;
continue;
}
let id = config::unique_slug(title, &cfg.feeds);
cfg.feeds.insert(
id.clone(),
config::Feed {
url: url.clone(),
// Nested, so the whole subscription lands in one folder.
folder: Some(format!("{base_folder}/{title}")),
group: Some(parent_id.to_owned()),
schedule: parent.schedule.clone(),
keywords: parent.keywords.clone(),
allow_explicit: parent.allow_explicit,
auto_download: parent.auto_download,
max_new_per_check: parent.max_new_per_check,
username: parent.username.clone(),
password: parent.password.clone(),
password_env: parent.password_env.clone(),
},
);
if cfg.feeds.values().any(|f| &f.url == url) {
continue;
}
let taken: std::collections::BTreeMap<String, config::Feed> = cfg
.feeds
.keys()
.chain(existing.iter().map(|m| &m.id))
.chain(added.iter())
.map(|id| (id.clone(), parent.clone()))
.collect();
let id = config::unique_slug(title, &taken);
ctx.db.upsert_managed(&id, url, title, parent_id)?;
added.push(id);
}
// Anything in this group the OPML no longer lists.
let gone: Vec<String> = cfg
.feeds
.iter()
.filter(|(_, f)| f.group.as_deref() == Some(parent_id))
.filter(|(_, f)| !listed.iter().any(|(_, u)| u == &f.url))
.map(|(id, _)| id.clone())
.collect();
let mut removed = 0;
let mut kept = 0;
for id in gone {
if ctx.db.downloaded_count(&id).unwrap_or(1) > 0 {
for m in existing.iter().filter(|m| m.group_id == parent_id) {
if listed.iter().any(|(_, u)| u == &m.url) {
continue;
}
if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 {
// Never orphan a downloaded file: keep the feed and say why in the UI.
ctx.db.set_orphaned(&id, true)?;
ctx.db.set_orphaned(&m.id, true)?;
kept += 1;
tracing::info!(feed = %id, "dropped from the OPML but has downloads; keeping it");
tracing::info!(feed = %m.id, "dropped from the OPML but has downloads; keeping it");
} else {
cfg.feeds.remove(&id);
ctx.db.drop_managed(&m.id)?;
removed += 1;
tracing::info!(feed = %id, "dropped from the OPML with nothing downloaded; unsubscribed");
tracing::info!(feed = %m.id, "dropped from the OPML with nothing downloaded; removed");
}
}
if !added.is_empty() || removed > 0 {
cfg.save(&ctx.config_path)?;
ctx.reload_cfg(&ctx.config_path)?;
}
Ok(Outcome::Opml { added, removed, kept, total: listed.len() })
}