diff --git a/PROGRESS.md b/PROGRESS.md index 8ad9a53..2bde1b7 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -56,6 +56,35 @@ and until now nothing set them. --- +## 2026-09-10 — OPML feeds out of the config, a real cap, and daemon output in the log + +Three reports in quick succession, all fair. + +**"Config is a mess."** It was: 611 lines, 85 feeds, 82 of them machine-generated, drowning the three +Ray actually chose. Writing derived data into a hand-edited file was the wrong call. OPML children +now live in the `feeds` table (`managed = 1`, `group_id`), are re-derived every scan, and inherit the +subscription's settings — so there is nothing to store but a URL and a parent. Editing one promotes +it to a real config entry, so `config.toml` only ever holds decisions. A one-time migration moves +existing children out: **611 lines -> 38**, 85 feeds -> 3, with all 3487 entries and 216 files intact. + +**"Never download more than 3."** `max_new_per_check` defaulted to *unlimited*, so subscribing to an +OPML of 82 feeds pulled every back catalogue it could reach — 216 files, 22 GB before it was caught. +There is now `[general] max_new_per_check = 3`, used whenever a feed does not set its own, so all 83 +uncapped feeds were capped without touching a line of their config. `pending()` also orders by publish +date now: a cap of 3 meant "the three recorded first", not the three newest. + +**"No daemon output in the log."** Scans and downloads travel as events to the socket, not through +tracing, so the log view showed only startup and HTTP lines. `Emitter::emit` mirrors them now — and +levelling matters at this scale: at 82 feeds, one INFO line per feed per tick for "not due yet" +flushed the 2000-line buffer of anything useful in minutes, so routine skips and progress are DEBUG, +real activity is INFO, failures WARN. + +A splice while refactoring cut `reject` and `fetch_one` out of main.rs, and recovering them from git +over-copied three more. Both caught by the compiler, restored, and verified byte-identical against +`git show HEAD:src/main.rs` rather than eyeballed. + +--- + ## 2026-09-10 — Subscribing to an OPML, not just importing one Asked whether this version could do what iPodderX did: subscribe to an OPML and get a folder of the diff --git a/README.md b/README.md index 632175b..bf2254e 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ interval_mins = 60 # default poll; a feed's own wins when longer organize = "feed" # "feed" | "date" max_total_gb = 50 # 0 = unlimited max_age_days = 30 # 0 = keep forever +max_new_per_check = 3 # per feed, per scan. 0 = unlimited (pulls whole back catalogues) [torrent] enabled = true @@ -109,10 +110,14 @@ Two different things, both supported: `ipx export subs.opml`, or the OPML button in the UI. **Subscribing to an OPML URL** is a live subscription, as iPodderX had. Add the OPML's URL like any -other feed; every scan re-reads it and keeps your feed list in step. Feeds it lists are added under -that subscription (`group = ""` in the config, grouped in the sidebar) and downloaded into -one nested folder. New ones are scanned in the same run rather than waiting for the next interval. -An OPML is recognised by its content, so a URL without a `.opml` extension still works. +other feed; every scan re-reads it and keeps your feed list in step. An OPML is recognised by its +content, so a URL without a `.opml` extension still works. + +The feeds inside it are **not written to `config.toml`** — the OPML is the source of truth, so they +are re-derived each scan and held in the database. Your config keeps only what you chose. They show +as a collapsible folder in the sidebar, download into one nested folder, and newly listed ones are +scanned in the same run rather than waiting for the next interval. They inherit the subscription's +settings; change anything on one and it gets its own config entry from then on. When a feed drops out of the OPML upstream: diff --git a/src/config.rs b/src/config.rs index a88559c..1f23e7d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -34,6 +34,10 @@ pub struct General { pub max_total_gb: f64, /// 0 = keep forever. pub max_age_days: u64, + /// How many new enclosures a single scan may take, when a feed does not say. + /// Unlimited by default was a trap: subscribing to an OPML of 80 feeds then pulled + /// every back-catalogue episode at once. 0 means unlimited, deliberately chosen. + pub max_new_per_check: usize, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] @@ -104,7 +108,7 @@ pub struct Feed { /// Overrides the global schedule for this feed. Same forms: "every 6h", "2d". #[serde(default, skip_serializing_if = "Option::is_none")] pub schedule: Option, - /// Cap on new downloads per scan. None = unlimited. + /// Cap on new downloads per scan for this feed. None follows `[general]`. #[serde(default, skip_serializing_if = "Option::is_none")] pub max_new_per_check: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -130,6 +134,7 @@ impl Default for General { organize: Organize::Feed, max_total_gb: 0.0, max_age_days: 0, + max_new_per_check: 3, } } } diff --git a/src/db.rs b/src/db.rs index 35522bf..f420478 100644 --- a/src/db.rs +++ b/src/db.rs @@ -23,7 +23,13 @@ CREATE TABLE IF NOT EXISTS feeds ( ttl_mins INTEGER, last_error TEXT, -- Came from a subscribed OPML that no longer lists it, but has downloads, so kept. - orphaned INTEGER NOT NULL DEFAULT 0 + orphaned INTEGER NOT NULL DEFAULT 0, + -- The OPML subscription this feed came from. + group_id TEXT, + -- 1 = derived from an OPML and not written to config.toml. Writing 80-odd generated + -- entries into a hand-edited file made it unreadable; the OPML is the source of + -- truth, so they are re-derived instead. Customising one promotes it to config. + managed INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE IF NOT EXISTS entries ( @@ -65,12 +71,24 @@ CREATE TABLE IF NOT EXISTS enclosures ( CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid); "; +/// A feed derived from an OPML subscription rather than written into the config. +#[derive(Debug, Clone)] +pub struct Managed { + pub id: String, + pub url: String, + pub title: Option, + pub group_id: String, + pub orphaned: bool, +} + /// Adds columns that later versions introduced. CREATE TABLE IF NOT EXISTS does nothing to /// a table that already exists, so an installed database needs them added explicitly. fn migrate(conn: &Connection) -> Result<()> { let wanted: &[(&str, &str, &str)] = &[ ("feeds", "image", "TEXT"), ("feeds", "orphaned", "INTEGER NOT NULL DEFAULT 0"), + ("feeds", "group_id", "TEXT"), + ("feeds", "managed", "INTEGER NOT NULL DEFAULT 0"), ("entries", "image", "TEXT"), ("entries", "duration", "INTEGER"), ("entries", "episode", "INTEGER"), @@ -355,8 +373,13 @@ impl Db { pub fn pending(&self, feed_id: &str, limit: usize) -> Result> { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( - "SELECT id, url, mime FROM enclosures - WHERE feed_id = ?1 AND state = 'pending' ORDER BY id LIMIT ?2", + // Newest first: a cap of 3 should mean the three latest episodes, not the + // three that happen to have been recorded first. + "SELECT x.id, x.url, x.mime FROM enclosures x + JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid + WHERE x.feed_id = ?1 AND x.state = 'pending' + ORDER BY coalesce(e.published, e.first_seen) DESC, x.id DESC + LIMIT ?2", )?; let rows = stmt .query_map(rusqlite::params![feed_id, limit as i64], |r| { @@ -725,6 +748,77 @@ impl Db { )?) } + /// Names a feed without touching its conditional-GET validators. An OPML subscription + /// takes its name from the document's own . + pub fn set_title(&self, feed_id: &str, title: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "UPDATE feeds SET title = ?2 WHERE id = ?1 AND coalesce(title, '') != ?2", + rusqlite::params![feed_id, title], + )?; + Ok(()) + } + + /// Records a feed that came from an OPML. Its settings are the parent's; only what + /// identifies it is stored. + pub fn upsert_managed( + &self, + id: &str, + url: &str, + title: &str, + group_id: &str, + ) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute( + "INSERT INTO feeds (id, url, title, group_id, managed, orphaned) + VALUES (?1, ?2, ?3, ?4, 1, 0) + ON CONFLICT(id) DO UPDATE SET + url = excluded.url, + title = coalesce(feeds.title, excluded.title), + group_id = excluded.group_id, + managed = 1, + orphaned = 0", + rusqlite::params![id, url, title, group_id], + )?; + Ok(()) + } + + /// Every feed derived from an OPML, whichever group. + pub fn managed_feeds(&self) -> Result<Vec<Managed>> { + let conn = self.conn.lock().unwrap(); + let mut stmt = conn.prepare( + "SELECT id, url, title, group_id, orphaned FROM feeds + WHERE managed = 1 AND group_id IS NOT NULL ORDER BY coalesce(title, id)", + )?; + Ok(stmt + .query_map([], |r| { + Ok(Managed { + id: r.get(0)?, + url: r.get(1)?, + title: r.get(2)?, + group_id: r.get(3)?, + orphaned: r.get::<_, i64>(4)? != 0, + }) + })? + .collect::<rusqlite::Result<Vec<_>>>()?) + } + + /// Forgets a derived feed entirely. Only for one with nothing downloaded. + pub fn drop_managed(&self, id: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute("DELETE FROM feeds WHERE id = ?1 AND managed = 1", [id])?; + conn.execute("DELETE FROM entries WHERE feed_id = ?1", [id])?; + conn.execute("DELETE FROM enclosures WHERE feed_id = ?1 AND path IS NULL", [id])?; + Ok(()) + } + + /// Stops treating a feed as derived, because it now has its own config entry. + pub fn unmanage(&self, id: &str) -> Result<()> { + let conn = self.conn.lock().unwrap(); + conn.execute("UPDATE feeds SET managed = 0 WHERE id = ?1", [id])?; + Ok(()) + } + pub fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> { let conn = self.conn.lock().unwrap(); conn.execute( @@ -825,6 +919,23 @@ mod tests { "search is case-insensitive and covers the description"); } + #[test] + fn pending_takes_the_latest_episodes_first() { + // A cap of 3 must mean the three newest, not the three recorded first. + let db = Db::memory().unwrap(); + db.exec_for_test( + "INSERT INTO entries (feed_id, guid, published, first_seen) VALUES + ('f','old',100,100), ('f','mid',200,200), ('f','new',300,300); + INSERT INTO enclosures (id, feed_id, guid, url, state) VALUES + (1,'f','old','u-old','pending'), + (2,'f','mid','u-mid','pending'), + (3,'f','new','u-new','pending');", + ) + .unwrap(); + let got: Vec<String> = db.pending("f", 2).unwrap().into_iter().map(|p| p.url).collect(); + assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later"); + } + #[test] fn a_restart_requeues_interrupted_downloads() { let db = Db::memory().unwrap(); diff --git a/src/ipc.rs b/src/ipc.rs index 9718884..f1337e6 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -73,9 +73,9 @@ impl Event { } Event::Error { msg } => format!("error: {msg}"), // Noise in a terminal; a UI still gets them on the socket. - Event::FeedStart { .. } | Event::TorrentDeferred { .. } | Event::ScanDone { .. } => { - return None; - } + Event::FeedStart { feed } => format!("{feed}: checking"), + Event::TorrentDeferred { feed, .. } => format!("{feed}: torrent deferred"), + Event::ScanDone { feeds } => format!("scan complete, {feeds} feed(s)"), }) } } @@ -119,6 +119,35 @@ impl Emitter { } pub fn emit(&self, e: Event) { + // Also log it. Scans and downloads travel as events, not tracing calls, so + // without this the log view shows only startup and HTTP lines and none of the + // work the daemon is actually doing. Progress goes to debug: it fires on every + // whole percent and would otherwise crowd everything else out of the buffer. + // Level by how much it matters. With 80-odd feeds in an OPML subscription, one + // line per feed per tick for "not due yet" would push everything worth reading + // out of the buffer within a few minutes. + let routine = match &e { + Event::Progress { .. } | Event::FeedSkip { .. } | Event::FeedStart { .. } => true, + Event::FeedDone { new, downloaded, failed, torrents, .. } => { + *new == 0 && *downloaded == 0 && *failed == 0 && *torrents == 0 + } + _ => false, + }; + let bad = matches!( + &e, + Event::FeedError { .. } | Event::DownloadError { .. } | Event::Error { .. } + ); + if let Some(line) = e.human() { + let line = line.trim(); + if bad { + tracing::warn!(target: "ipx::scan", "{line}"); + } else if routine { + tracing::debug!(target: "ipx::scan", "{line}"); + } else { + tracing::info!(target: "ipx::scan", "{line}"); + } + } + if let Some(tx) = &self.tx { // An error here only means nobody is listening yet. let _ = tx.send(e.clone()); diff --git a/src/main.rs b/src/main.rs index 029241b..852571d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -217,6 +217,12 @@ async fn daemon( anyhow::bail!("a daemon is already listening on {}", socket.display()); } + match migrate_opml_children(&ctx) { + Ok(n) if n > 0 => tracing::info!(count = n, "moved OPML feeds out of config.toml into the database"), + Ok(_) => {} + Err(e) => tracing::warn!(error = ?e, "could not tidy OPML feeds out of the config"), + } + match ctx.db.requeue_interrupted() { Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"), Ok(_) => {} @@ -549,19 +555,17 @@ fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> { async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { let cfg = ctx.cfg(); + let subs = subscriptions(ctx)?; if let Some(id) = only - && !cfg.feeds.contains_key(id) + && !subs.iter().any(|s| s.id == id) { anyhow::bail!("no feed with id {id:?}"); } let mut scanned = 0; let mut fresh: Vec<String> = vec![]; - for (id, feed_cfg) in cfg - .feeds - .iter() - .filter(|(id, _)| only.is_none_or(|o| o == *id)) - { + for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) { + let (id, feed_cfg) = (&sub.id, &sub.cfg); let state = ctx.db.http_state(id)?; if !force && let Some(last) = state.last_checked { @@ -611,9 +615,11 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { } // Feeds a subscribed OPML just introduced: scan them now, in this run. if !fresh.is_empty() { - let cfg = ctx.cfg(); + let subs = subscriptions(ctx)?; for id in &fresh { - let Some(feed_cfg) = cfg.feeds.get(id) else { continue }; + let Some(feed_cfg) = subs.iter().find(|s| &s.id == id).map(|s| &s.cfg) else { + continue; + }; scanned += 1; ctx.out.emit(Event::FeedStart { feed: id.clone() }); let state = ctx.db.http_state(id)?; @@ -638,6 +644,89 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { Ok(()) } +/// One feed to scan: either an entry you wrote in config.toml, or one derived from an +/// OPML subscription and held only in the database. +pub struct Sub { + pub id: String, + pub cfg: config::Feed, + /// True when it came from an OPML and has no config entry of its own. + pub managed: bool, +} + +/// Everything to scan: your config entries, plus whatever the OPML subscriptions listed. +/// +/// A derived feed borrows its parent's settings wholesale. That is why it needs no config +/// entry -- there is nothing to store but its URL and where it came from. +pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { + let cfg = ctx.cfg(); + let mut out: Vec<Sub> = cfg + .feeds + .iter() + .map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false }) + .collect(); + + for m in ctx.db.managed_feeds()? { + if cfg.feeds.contains_key(&m.id) { + continue; // promoted to config at some point; that entry wins + } + let parent = cfg.feeds.get(&m.group_id); + let base = parent + .and_then(|p| p.folder.clone()) + .or_else(|| ctx.db.feed_summary(&m.group_id).ok().and_then(|s| s.title)) + .unwrap_or_else(|| m.group_id.clone()); + let title = m.title.clone().unwrap_or_else(|| m.id.clone()); + out.push(Sub { + id: m.id.clone(), + cfg: config::Feed { + url: m.url.clone(), + folder: Some(format!("{base}/{title}")), + group: Some(m.group_id.clone()), + schedule: parent.and_then(|p| p.schedule.clone()), + keywords: parent.map(|p| p.keywords.clone()).unwrap_or_default(), + allow_explicit: parent.is_some_and(|p| p.allow_explicit), + auto_download: parent.is_none_or(|p| p.auto_download), + max_new_per_check: parent.and_then(|p| p.max_new_per_check), + username: parent.and_then(|p| p.username.clone()), + password: parent.and_then(|p| p.password.clone()), + password_env: parent.and_then(|p| p.password_env.clone()), + }, + managed: true, + }); + } + out.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(out) +} + +/// Moves OPML children that older versions wrote into config.toml over to the database. +/// They were never yours to edit, and 80-odd of them made the file unreadable. +fn migrate_opml_children(ctx: &Ctx) -> Result<usize> { + let cfg = (*ctx.cfg()).clone(); + let children: Vec<(String, config::Feed)> = cfg + .feeds + .iter() + .filter(|(_, f)| f.group.is_some()) + .map(|(id, f)| (id.clone(), f.clone())) + .collect(); + if children.is_empty() { + return Ok(0); + } + let mut fresh = cfg.clone(); + for (id, f) in &children { + let group = f.group.clone().unwrap_or_default(); + let title = ctx + .db + .feed_summary(id) + .ok() + .and_then(|s| s.title) + .unwrap_or_else(|| id.clone()); + ctx.db.upsert_managed(id, &f.url, &title, &group)?; + fresh.feeds.remove(id); + } + fresh.save(&ctx.config_path)?; + ctx.reload_cfg(&ctx.config_path)?; + Ok(children.len()) +} + /// Seconds to wait before re-checking a feed. /// /// A per-feed schedule is an explicit instruction and wins outright. Without one, the @@ -724,7 +813,11 @@ async fn scan_one( } } - let budget = feed_cfg.max_new_per_check.unwrap_or(usize::MAX); + // An unset per-feed cap follows the global one; 0 there means unlimited. + let budget = feed_cfg.max_new_per_check.unwrap_or_else(|| { + let g = ctx.cfg().general.max_new_per_check; + if g == 0 { usize::MAX } else { g } + }); if feed_cfg.auto_download && budget > 0 { let cfg = ctx.cfg(); let folder = download::folder_for(&cfg, id, feed_cfg, parsed.title.as_deref()); @@ -816,71 +909,54 @@ async fn sync_opml( bytes: &[u8], ) -> Result<Outcome> { let listed = feed::parse_opml(bytes)?; - let mut cfg = (*ctx.cfg()).clone(); - - let base_folder = parent - .folder - .clone() - .or_else(|| ctx.db.feed_summary(parent_id).ok().and_then(|s| s.title)) - .unwrap_or_else(|| parent_id.to_owned()); + if let Some(title) = feed::opml_title(bytes) { + ctx.db.set_title(parent_id, &title)?; + } + let cfg = ctx.cfg(); + let existing = ctx.db.managed_feeds()?; let mut added = vec![]; + for (title, url) in &listed { - if let Some((id, _)) = cfg.feeds.iter().find(|(_, f)| &f.url == url) { - // Already subscribed. If it had been flagged as gone, it is back. - let id = id.clone(); - let _ = ctx.db.set_orphaned(&id, false); + // Already known, whether derived or promoted into the config. + if let Some(m) = existing.iter().find(|m| &m.url == url) { + ctx.db.upsert_managed(&m.id, url, title, parent_id)?; continue; } - let id = config::unique_slug(title, &cfg.feeds); - cfg.feeds.insert( - id.clone(), - config::Feed { - url: url.clone(), - // Nested, so the whole subscription lands in one folder. - folder: Some(format!("{base_folder}/{title}")), - group: Some(parent_id.to_owned()), - schedule: parent.schedule.clone(), - keywords: parent.keywords.clone(), - allow_explicit: parent.allow_explicit, - auto_download: parent.auto_download, - max_new_per_check: parent.max_new_per_check, - username: parent.username.clone(), - password: parent.password.clone(), - password_env: parent.password_env.clone(), - }, - ); + if cfg.feeds.values().any(|f| &f.url == url) { + continue; + } + let taken: std::collections::BTreeMap<String, config::Feed> = cfg + .feeds + .keys() + .chain(existing.iter().map(|m| &m.id)) + .chain(added.iter()) + .map(|id| (id.clone(), parent.clone())) + .collect(); + let id = config::unique_slug(title, &taken); + ctx.db.upsert_managed(&id, url, title, parent_id)?; added.push(id); } // Anything in this group the OPML no longer lists. - let gone: Vec<String> = cfg - .feeds - .iter() - .filter(|(_, f)| f.group.as_deref() == Some(parent_id)) - .filter(|(_, f)| !listed.iter().any(|(_, u)| u == &f.url)) - .map(|(id, _)| id.clone()) - .collect(); - let mut removed = 0; let mut kept = 0; - for id in gone { - if ctx.db.downloaded_count(&id).unwrap_or(1) > 0 { + for m in existing.iter().filter(|m| m.group_id == parent_id) { + if listed.iter().any(|(_, u)| u == &m.url) { + continue; + } + if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 { // Never orphan a downloaded file: keep the feed and say why in the UI. - ctx.db.set_orphaned(&id, true)?; + ctx.db.set_orphaned(&m.id, true)?; kept += 1; - tracing::info!(feed = %id, "dropped from the OPML but has downloads; keeping it"); + tracing::info!(feed = %m.id, "dropped from the OPML but has downloads; keeping it"); } else { - cfg.feeds.remove(&id); + ctx.db.drop_managed(&m.id)?; removed += 1; - tracing::info!(feed = %id, "dropped from the OPML with nothing downloaded; unsubscribed"); + tracing::info!(feed = %m.id, "dropped from the OPML with nothing downloaded; removed"); } } - if !added.is_empty() || removed > 0 { - cfg.save(&ctx.config_path)?; - ctx.reload_cfg(&ctx.config_path)?; - } Ok(Outcome::Opml { added, removed, kept, total: listed.len() }) } diff --git a/src/web.rs b/src/web.rs index 5677378..df22fca 100644 --- a/src/web.rs +++ b/src/web.rs @@ -159,6 +159,7 @@ struct FeedRow { group: Option<String>, /// In a group, but the OPML no longer lists it. Kept because it has downloads. orphaned: bool, + managed: bool, schedule: Option<String>, /// The feed's own override in minutes, so the UI need not re-parse the string. schedule_mins: Option<u64>, @@ -174,8 +175,11 @@ struct FeedRow { async fn feeds(State(state): State<WebState>) -> Result<Json<Vec<FeedRow>>, ApiError> { let cfg = state.ctx.cfg(); - let mut out = Vec::with_capacity(cfg.feeds.len()); - for (id, feed) in &cfg.feeds { + // Config entries plus the feeds derived from OPML subscriptions. + let subs = crate::subscriptions(&state.ctx)?; + let mut out = Vec::with_capacity(subs.len()); + for sub in &subs { + let (id, feed) = (&sub.id, &sub.cfg); let s = state.ctx.db.feed_summary(id)?; let st = state.ctx.db.http_state(id)?; out.push(FeedRow { @@ -190,6 +194,8 @@ async fn feeds(State(state): State<WebState>) -> Result<Json<Vec<FeedRow>>, ApiE max_new_per_check: feed.max_new_per_check, group: feed.group.clone(), orphaned: s.orphaned, + // Derived from an OPML and not written to config until you change something. + managed: sub.managed, schedule: feed.schedule.clone(), schedule_mins: feed .schedule @@ -435,6 +441,18 @@ async fn patch_feed( ) -> Result<StatusCode, ApiError> { let mut cfg = (*state.ctx.cfg()).clone(); + // Derived feeds have no config entry. Editing one is the moment it earns a real + // entry: promote it, so the config holds your decisions and nothing else. + if !cfg.feeds.contains_key(&id) { + let subs = crate::subscriptions(&state.ctx)?; + let found = subs + .iter() + .find(|s| s.id == id) + .ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?; + cfg.feeds.insert(id.clone(), found.cfg.clone()); + state.ctx.db.unmanage(&id)?; + } + let checked = match &body.url { Some(u) => Some(check_url(u, &id, &cfg.feeds).map_err(ApiError::bad_request)?), None => None, @@ -493,7 +511,10 @@ async fn remove_feed( ) -> Result<StatusCode, ApiError> { let mut cfg = (*state.ctx.cfg()).clone(); if cfg.feeds.remove(&id).is_none() { - return Err(anyhow::anyhow!("no feed with id {id:?}").into()); + // A derived feed: forget it here, though the OPML will list it again on the next + // read unless you unsubscribe from the OPML itself. + state.ctx.db.drop_managed(&id)?; + return Ok(StatusCode::NO_CONTENT); } // Downloads and history stay, so re-adding does not re-pull the back catalogue. cfg.save(&state.config_path)?; @@ -750,6 +771,7 @@ async fn import_opml( struct Settings { schedule: String, every_mins: u64, + max_new_per_check: usize, download_dir: String, max_total_gb: f64, max_age_days: u64, @@ -760,6 +782,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> { Json(Settings { schedule: cfg.general.schedule.clone(), every_mins: cfg.general.interval(), + max_new_per_check: cfg.general.max_new_per_check, download_dir: cfg.general.download_dir.display().to_string(), max_total_gb: cfg.general.max_total_gb, max_age_days: cfg.general.max_age_days, @@ -769,6 +792,7 @@ async fn get_settings(State(state): State<WebState>) -> Json<Settings> { #[derive(Deserialize)] struct SettingsPatch { schedule: Option<String>, + max_new_per_check: Option<usize>, max_total_gb: Option<f64>, max_age_days: Option<u64>, } @@ -789,6 +813,9 @@ async fn patch_settings( // The legacy key would otherwise keep shadowing intent in the file. cfg.general.interval_mins = None; } + if let Some(v) = body.max_new_per_check { + cfg.general.max_new_per_check = v; + } if let Some(v) = body.max_total_gb { cfg.general.max_total_gb = v.max(0.0); } diff --git a/tests/page-smoke.js b/tests/page-smoke.js index f11a1eb..01d6ec6 100644 --- a/tests/page-smoke.js +++ b/tests/page-smoke.js @@ -80,6 +80,9 @@ const drive = [ ['removeFeed', () => ctx.removeFeed(feed)], ['prefsModal', () => ctx.prefsModal()], ['logsModal', () => ctx.logsModal()], + // `const S` is not reachable from here: top-level const/let do not become properties + // of a vm context the way var and function declarations do. + ['renderGroup', () => ctx.renderGroup(feed, [{ ...feed, id: 'child', group: 'f', orphaned: true }])], ]; for (const [name, fn] of drive) { try { diff --git a/web/index.html b/web/index.html index 52f3e8d..b0586b7 100644 --- a/web/index.html +++ b/web/index.html @@ -94,7 +94,21 @@ input:focus,select:focus{outline:0;border-color:var(--accent)} .feed.sel{background:var(--raise)} .feed.child{margin-left:14px} .feed.child .art{width:28px;height:28px;font-size:11px} -.feed.group>.txt>b::after{content:" ⌄";color:var(--faint);font-weight:400} +.chev{ + flex:none;width:18px;height:18px;display:grid;place-items:center;border-radius:4px; + color:var(--faint);font-size:11px;transition:transform .12s; +} +.chev:hover{background:var(--raise);color:var(--fg)} +.chev.open{transform:rotate(90deg)} +.childlist{display:grid;gap:4px;margin-top:10px} +.childrow{ + display:flex;gap:10px;align-items:center;padding:7px 9px;border:1px solid var(--line); + border-radius:9px;cursor:pointer;background:var(--panel); +} +.childrow:hover{border-color:var(--accent)} +.childrow .art{width:32px;height:32px;font-size:12px} +.childrow .txt{flex:1;min-width:0} +.childrow .txt b{display:block;font-size:13.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} .tag{ font-size:10px;text-transform:uppercase;letter-spacing:.04em;font-weight:700; padding:1px 5px;border-radius:4px;background:var(--raise);color:var(--warn);flex:none; @@ -402,7 +416,7 @@ const LIMIT = 50; /* ---------------- feeds ---------------- */ async function loadFeeds(keepSel){ S.feeds = await api('/api/feeds'); - api('/api/settings').then(g=>{globalEvery=g.every_mins}).catch(()=>{}); + api('/api/settings').then(g=>{globalEvery=g.every_mins;globalMax=g.max_new_per_check}).catch(()=>{}); renderFeeds(); if(!keepSel && !S.feed && S.feeds.length) selectFeed(S.feeds[0].id); } @@ -418,19 +432,26 @@ function renderFeeds(){ for(const f of shown){ if(f.group && byId[f.group]) continue; // drawn under its parent instead order.push([f,0]); - for(const c of shown) if(c.group===f.id) order.push([c,1]); + // A subscription can hold dozens of feeds, so the folder starts closed. + if(!collapsed.has(f.id) || q) for(const c of shown) if(c.group===f.id) order.push([c,1]); } for(const [f,depth] of order){ const kids=shown.filter(c=>c.group===f.id).length; const el=document.createElement('div'); el.className='feed'+(S.feed===f.id?' sel':'')+(depth?' child':'')+(kids?' group':''); - el.innerHTML = artHTML(f.image,f.title||f.id)+ + const open = kids && (!collapsed.has(f.id) || q); + el.innerHTML = + (kids?`<span class="chev${open?' open':''}" title="Show or hide the feeds inside">▶</span>`:'')+ + artHTML(f.image,f.title||f.id)+ `<div class="txt"><b>${esc(f.title||f.id)}</b><small>`+ - (kids?`${kids} feed${kids===1?'':'s'}`:`${f.entries} eps · ${f.downloaded} saved`)+ + (kids?`${kids} feed${kids===1?'':'s'}${f.unread?` · ${f.unread} unread`:''}` + :`${f.entries} eps · ${f.downloaded} saved`)+ `</small></div>`+ (f.orphaned?'<span class="tag" title="No longer listed in the OPML, kept because it has downloads">gone</span>':'')+ `<span class="badge${f.unread?'':' zero'}">${f.unread}</span>`; el.onclick=()=>{ selectFeed(f.id); $('#sidebar').classList.remove('open'); }; + const chev=$('.chev',el); + if(chev) chev.onclick=ev=>{ ev.stopPropagation(); toggleGroup(f.id); }; list.appendChild(el); } } @@ -443,6 +464,8 @@ function selectFeed(id){ function renderFeed(){ const f=S.feeds.find(x=>x.id===S.feed); if(!f){ $('#content').innerHTML='<p class="empty">Add a feed to get started.</p>'; return; } + const kids=S.feeds.filter(c=>c.group===f.id); + if(kids.length){ renderGroup(f,kids); return; } $('#content').innerHTML = ` <div class="fhead"> ${artHTML(f.image,f.title||f.id)} @@ -478,6 +501,58 @@ function renderFeed(){ let t; $('#epSearch').oninput=e=>{clearTimeout(t);t=setTimeout(()=>{S.q=e.target.value;S.offset=0;loadEntries()},250)}; } +/// An OPML subscription's page lists the feeds inside it rather than episodes, but keeps +/// every action a normal feed has -- it is still an ordinary feed entry underneath. +function renderGroup(f,kids){ + const unread=kids.reduce((n,c)=>n+c.unread,0); + const saved=kids.reduce((n,c)=>n+c.downloaded,0); + const gone=kids.filter(c=>c.orphaned).length; + $('#content').innerHTML = ` + <div class="fhead"> + ${artHTML(f.image,f.title||f.id)} + <div class="meta"> + <h2>${esc(f.title||f.id)}</h2> + <div class="sub">OPML subscription · ${kids.length} feed${kids.length===1?'':'s'} + · ${unread} unread · ${saved} downloaded · checked ${ago(f.last_checked)} + · every ${everyText(f.every_mins)}</div> + ${f.last_error?`<div class="sub" style="color:var(--bad)">${esc(f.last_error)}</div>`:''} + ${gone?`<div class="sub" style="color:var(--warn)">${gone} feed${gone===1?' is':'s are'} no longer + listed in this OPML but kept because ${gone===1?'it has':'they have'} downloads.</div>`:''} + <div class="acts"> + <button class="btn primary" data-a="scan">Re-read OPML</button> + <button class="btn" data-a="settings">Settings</button> + <button class="btn danger" data-a="rm">Unsubscribe</button> + </div> + </div> + </div> + <div class="toolbar"> + <input type="search" class="grow" id="kidSearch" placeholder="Search these feeds…"> + <span style="color:var(--faint);font-size:12.5px">${esc(f.url)}</span> + </div> + <div class="childlist" id="kidlist"></div>`; + $$('#content .acts .btn').forEach(b=>b.onclick=()=>feedAction(b.dataset.a,f)); + const draw=()=>{ + const q=($('#kidSearch').value||'').trim().toLowerCase(); + const box=$('#kidlist'); box.innerHTML=''; + const rows=kids.filter(c=>!q||(c.title||c.id).toLowerCase().includes(q)); + if(!rows.length){ box.innerHTML='<p class="empty">Nothing matches.</p>'; return; } + for(const c of rows){ + const el=document.createElement('div'); + el.className='childrow'; + el.innerHTML = artHTML(c.image,c.title||c.id)+ + `<div class="txt"><b>${esc(c.title||c.id)}</b>`+ + `<small class="meta">${c.entries} eps · ${c.downloaded} downloaded`+ + (c.last_error?` · <span style="color:var(--bad)">error</span>`:'')+`</small></div>`+ + (c.orphaned?'<span class="tag">gone</span>':'')+ + `<span class="badge${c.unread?'':' zero'}">${c.unread}</span>`; + el.onclick=()=>selectFeed(c.id); + box.appendChild(el); + } + }; + $('#kidSearch').oninput=draw; + draw(); +} + async function feedAction(a,f){ if(a==='scan'){ toast('Scanning '+(f.title||f.id)+'…'); await api('/api/fetch',{method:'POST',body:JSON.stringify({feed:f.id,force:true})}); } if(a==='read'){ const r=await api(`/api/feeds/${encodeURIComponent(f.id)}/read-all`,{method:'POST'}); toast(`Marked ${r.marked} read`); await loadFeeds(true); loadEntries(); } @@ -780,7 +855,13 @@ $('#addFeed').onclick=()=>{ }; }; -let globalEvery = 60; +let collapsed = new Set(JSON.parse(localStorage.getItem('ipx.collapsed')||'[]')); +function toggleGroup(id){ + collapsed.has(id) ? collapsed.delete(id) : collapsed.add(id); + try{ localStorage.setItem('ipx.collapsed', JSON.stringify([...collapsed])); }catch{} + renderFeeds(); +} +let globalEvery = 60, globalMax = 3; const UNITS = [['m','minutes'],['h','hours'],['d','days'],['w','weeks']]; const UNIT_MINS = {m:1, h:60, d:1440, w:10080}; @@ -822,6 +903,11 @@ async function prefsModal(){ </div> <span class="hint">Applies to every feed that does not set its own. A feed's suggested interval (its <b>ttl</b>) is still honoured when it asks to be polled less often.</span></div> + <div class="field"><label>Max new downloads per scan, per feed</label> + <input type="number" id="gmax" min="0" max="999" value="${g.max_new_per_check}"> + <span class="hint">Applies to any feed that does not set its own — including every feed + inside an OPML subscription. <b>0 means unlimited</b>, which will pull a whole back + catalogue the first time a feed is scanned.</span></div> <div class="field"><label>Disk quota (GB, 0 = unlimited)</label> <input type="number" id="gquota" min="0" step="0.5" value="${g.max_total_gb}"> <span class="hint">Over this, the oldest played episodes are deleted first. Starred @@ -836,6 +922,7 @@ async function prefsModal(){ try{ await api('/api/settings',{method:'PATCH',body:JSON.stringify({ schedule:`every ${Math.max(1,Number($('#gnum').value)||1)}${$('#gunit').value}`, + max_new_per_check:Math.max(0,Number($('#gmax').value)||0), max_total_gb:Number($('#gquota').value)||0, max_age_days:Number($('#gage').value)||0})}); closeModal(); toast('Settings saved'); @@ -846,7 +933,13 @@ async function prefsModal(){ function settingsModal(f){ const fs = splitEvery(f.schedule_mins || globalEvery); + const isGroup = S.feeds.some(c=>c.group===f.id); openModal(`<h3>${esc(f.title||f.id)}</h3> + ${isGroup?`<p class="hint" style="margin:-6px 0 12px">This is an OPML subscription. These + settings apply to it and are inherited by every feed inside it.</p>`:''} + ${f.managed?`<p class="hint" style="margin:-6px 0 12px">This feed comes from an OPML + subscription and follows its settings. Saving anything here gives it its own entry in + config.toml, and it stops following the subscription's settings.</p>`:''} <div class="field"><label>Download folder</label> <input type="text" id="sfolder" value="${esc(f.folder||'')}" placeholder="${esc(f.title||f.id)}"></div> <div class="field"><label>Keywords</label> @@ -861,7 +954,8 @@ function settingsModal(f){ interval, for this feed only.</span></div> <div class="field"><label>Max new downloads per scan</label> <input type="number" id="smax" min="0" value="${f.max_new_per_check??''}"> - <span class="hint">Blank means no limit. The rest wait for the next scan.</span></div> + <span class="hint">Blank follows the global default (${globalMax}). The rest wait for + the next scan.</span></div> <label class="check"><input type="checkbox" id="sauto" ${f.auto_download?'checked':''}> Download new episodes automatically</label> <label class="check"><input type="checkbox" id="sexp" ${f.allow_explicit?'checked':''}> Allow episodes marked explicit</label> <div class="field"><label>Feed URL</label>