diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f162ad..7c47c72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `ipx add --list --category News ` puts a feed in the Directory for anyone to subscribe to, + and it stays there when its last subscriber leaves. +- Feeds nobody subscribes to that are dead for a month or quiet for a year are removed, so the + Directory lists feeds worth taking. + +### Changed + +- The Directory lists feeds nobody subscribes to yet; Popular still lists what people subscribe to. + A feed nobody subscribes to is checked once a day, and at once when someone subscribes. +- The Directory loads faster: it asked the database three questions per feed. + ## [0.9.1] - 2026-09-29 ### Added diff --git a/docs/configuration.md b/docs/configuration.md index 79ac544..6797d24 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -124,6 +124,7 @@ folder = "Accidental Tech Podcast" # default: the feed title schedule = "every 6h" # overrides [general] for this feed media_types = ["audio"] # overrides [general] for this feed category = "Technology" # the Directory's, if the feed names none +listed = true # in the Directory with no subscribers username = "ray" # HTTP basic auth password_env = "IPX_ATP_PASS" # preferred over a literal `password` ``` @@ -133,6 +134,12 @@ With more than one account, **`keywords`, `auto_download`, `allow_explicit` and config.toml are the fallback for a feed nobody has claimed. The keys above describe the feed itself and are the same for everyone. See [users.md](users.md). +**The Directory** lists every feed in the catalogue, whether or not anyone subscribes to it yet. +To put one there for others to find, `ipx add --list --category News `: it subscribes no one, +and a listed feed stays when its last subscriber leaves, where another is dropped. A feed nobody +subscribes to is checked once a day. One that is dead (failing for 30 days) or quiet (nothing +new in a year), with nobody subscribed and no file on disk, is removed from the catalogue. + Feeds derived from a subscribed OPML are **not** in the catalogue: the OPML is the source of truth and they are re-derived on every scan. Editing one in the UI promotes it to a catalogue entry. diff --git a/src/config.rs b/src/config.rs index 2e05a20..2bf6040 100644 --- a/src/config.rs +++ b/src/config.rs @@ -163,6 +163,10 @@ pub struct Feed { /// Preferred over `password`: name of an env var holding the password. #[serde(default, skip_serializing_if = "Option::is_none")] pub password_env: Option, + /// Put in the Directory by an admin (`ipx add --list`): listed with no subscribers, and kept + /// in the catalogue when the last one leaves, where anyone else's feed is dropped. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub listed: bool, } fn yes() -> bool { @@ -584,7 +588,7 @@ mod tests { taken.insert("the-daily".to_string(), Feed { url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, - password: None, password_env: None, category: None, + password: None, password_env: None, category: None, listed: false, }); assert_eq!(unique_slug("The Daily", &taken), "the-daily-2"); } @@ -605,6 +609,7 @@ mod tests { password: Some("literal".into()), password_env: None, category: None, + listed: false, }; assert_eq!(f.password().as_deref(), Some("literal")); diff --git a/src/db.rs b/src/db.rs index 7363ad3..b01629a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -778,13 +778,47 @@ impl Db { .map(|r| Ok(r.try_get("", "id")?)) .collect::>()?; for id in &gone { - for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] { - self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.clone().into()]).await?; - } + self.forget_rows(id).await?; } Ok(gone) } + /// Everything stored about a feed but its row: items, file rows, read state, block list. + async fn forget_rows(&self, id: &str) -> Result<()> { + for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] { + self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.into()]).await?; + } + Ok(()) + } + + /// Forgets a feed and everything stored about it. + pub async fn forget_feed(&self, id: &str) -> Result<()> { + self.forget_rows(id).await?; + self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?; + Ok(()) + } + + /// Feeds nobody subscribes to that are dead, failing since before `dead_before`, or quiet, + /// their newest item from before `quiet_before`: what keeps the Directory to feeds worth + /// taking. One with a file on disk stays, and so does one an OPML subscription lists, which + /// comes back with the OPML anyway. A feed with no items yet is not quiet, only new. + pub async fn stale_unsubscribed(&self, dead_before: i64, quiet_before: i64) -> Result> { + self.rows( + "SELECT f.id FROM feeds f + WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id) + AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = f.id AND x.path IS NOT NULL) + AND f.group_id IS NULL AND NOT coalesce(f.managed, false) + AND ((f.error_since IS NOT NULL AND f.error_since < $1) + OR (SELECT max(coalesce(e.published, e.first_seen)) FROM entries e + WHERE e.feed_id = f.id) < $2)", + vec![dead_before.into(), quiet_before.into()], + ) + .await? + .iter() + .map(|r| Ok(r.try_get("", "id")?)) + .collect() + } + pub async fn prune_entries(&self, older_than: i64) -> Result { let n = self .exec( @@ -1998,6 +2032,32 @@ mod tests { assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's } + #[tokio::test] + async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() { + let db = Db::memory().await.unwrap(); + db.exec_for_test( + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true); + INSERT INTO feeds (id, url, error_since, managed, group_id) VALUES + ('dead','a',100,false,null),('quiet','b',null,false,null),('news','c',null,false,null), + ('new','d',null,false,null),('wanted','e',100,false,null),('kept','f',100,false,null), + ('child','g',100,true,'opml'),('lately','h',900,false,null); + INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted'); + INSERT INTO entries (feed_id, guid, first_seen, published) VALUES + ('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100); + INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');", + ).await + .unwrap(); + // Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100. + // news published lately, new has no items yet, wanted has a subscriber, kept a file, + // child comes from an OPML, lately began failing after the cutoff. + let mut stale = db.stale_unsubscribed(500, 500).await.unwrap(); + stale.sort(); + assert_eq!(stale, ["dead", "quiet"]); + db.forget_feed("dead").await.unwrap(); + assert_eq!(db.feed_summary("dead").await.unwrap().entries, 0); + assert!(!db.feed_urls().await.unwrap().iter().any(|(id, _)| id == "dead")); + } + #[tokio::test] async fn a_failing_feed_nobody_subscribes_to_is_forgotten() { let db = Db::memory().await.unwrap(); diff --git a/src/download.rs b/src/download.rs index ad8251d..4e2e711 100644 --- a/src/download.rs +++ b/src/download.rs @@ -406,7 +406,7 @@ mod tests { let mut f = crate::config::Feed { url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, - max_new_per_check: None, username: None, password: None, password_env: None, category: None, + max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false, }; assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show"); diff --git a/src/main.rs b/src/main.rs index ee5d804..140f0bc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,12 @@ enum Command { /// Only take enclosures matching these keywords #[arg(long, value_delimiter = ',')] keywords: Vec, + /// The Directory's category, for a feed that names none of its own (News, Technology) + #[arg(long)] + category: Option, + /// Put it in the Directory for anyone to subscribe to, and keep it there when nobody does + #[arg(long)] + list: bool, }, /// Unsubscribe. Downloads and history are left alone. Rm { feed: String }, @@ -259,8 +265,8 @@ async fn main() -> Result<()> { let result = match cli.command { Command::List => list(&ctx).await, Command::Daemon { web } => daemon(ctx, config_path, web, events).await, - Command::Add { url, folder, keywords } => { - add(&ctx, &url, folder, keywords).await + Command::Add { url, folder, keywords, category, list } => { + add(&ctx, &url, folder, keywords, category, list).await } Command::Rm { feed } => rm(&ctx, &feed).await, Command::User { cmd } => user_cmd(&ctx, cmd).await, @@ -673,6 +679,8 @@ async fn add( url: &str, folder: Option, keywords: Vec, + category: Option, + list: bool, ) -> Result<()> { let mut cfg = (*ctx.cfg()).clone(); let url = &feed::find_feed(&ctx.client, &feed::expand_input(url)).await?; @@ -681,8 +689,12 @@ async fn add( anyhow::bail!("already subscribed as {:?}", existing.id); } let id = add_one(ctx, &mut cfg, url, folder, keywords).await?; + if let Some(f) = cfg.feeds.get_mut(&id) { + f.category = category; + f.listed = list; + } ctx.store_cfg(cfg).await?; - println!("added {id}"); + println!("added {id}{}", if list { ", listed in the Directory" } else { "" }); Ok(()) } @@ -709,6 +721,7 @@ pub async fn add_one( password: None, password_env: None, category: None, + listed: false, }; let title = match feed::fetch(&ctx.client, &probe, None, None).await { @@ -835,6 +848,7 @@ pub async fn subscribe_opml( password: None, password_env: None, category: None, + listed: false, }, ); grew = true; @@ -947,12 +961,38 @@ async fn list(ctx: &Ctx) -> Result<()> { Ok(()) } +/// Removes feeds nobody subscribes to that are dead (failing for 30 days) or quiet (nothing new +/// in a year), from the catalogue and the database, so the Directory lists feeds worth taking. +/// A feed listed in it with no subscribers stays for as long as it works and publishes. +async fn clean_directory(ctx: &Ctx) -> Result<()> { + const DEAD: i64 = 30 * 86_400; + const QUIET: i64 = 365 * 86_400; + let now = db::now(); + let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?; + if stale.is_empty() { + return Ok(()); + } + let mut cfg = (*ctx.cfg()).clone(); + if stale.iter().fold(false, |any, id| cfg.feeds.remove(id).is_some() || any) { + ctx.store_cfg(cfg).await?; + } + for id in &stale { + retire_group(ctx, id).await?; + ctx.db.forget_feed(id).await?; + tracing::info!(feed = id, "removed a feed nobody subscribes to that is dead or has published nothing in a year"); + } + Ok(()) +} + /// `standalone` false means this is the sweep that runs before a scan: it reports what it /// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch` /// would stop reading before the scan had even started. #[tracing::instrument(name = "reap", skip_all, fields(dry_run = dry_run))] async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> { let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?; + if !dry_run { + clean_directory(ctx).await?; + } for c in r.aged_out.iter().chain(r.over_quota.iter()) { ctx.out.emit(Event::Reaped { path: c.path.clone(), @@ -979,6 +1019,7 @@ async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope: &[String] anyhow::bail!("no feed with id {id:?}"); } + let subscribed = ctx.db.subscriber_counts().await?; let mut scanned = 0; let mut fresh: Vec = vec![]; let in_scope = |s: &Sub| { @@ -996,7 +1037,11 @@ async fn fetch(ctx: &Arc, only: Option<&str>, force: bool, scope: &[String] } if !force && let Some(last) = state.last_checked { - let at = last + due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last) as i64; + // A feed nobody subscribes to, one listed in the Directory, is read once a day: enough + // to keep its entry current, without fetching it hourly for no one. + let floor = if subscribed.contains_key(id) { 0 } else { 86_400 }; + let wait = due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last).max(floor); + let at = last + wait as i64; if at > db::now() { ctx.out.emit(Event::FeedSkip { feed: id.clone(), @@ -1156,6 +1201,7 @@ pub async fn subscriptions(ctx: &Ctx) -> Result> { password: parent.and_then(|p| p.password.clone()), password_env: parent.and_then(|p| p.password_env.clone()), category: None, + listed: false, }, managed: true, }); @@ -1969,6 +2015,7 @@ mod tests { password: None, password_env: None, category: None, + listed: false, } } diff --git a/src/web.rs b/src/web.rs index d9484b2..3ad895a 100644 --- a/src/web.rs +++ b/src/web.rs @@ -870,8 +870,13 @@ struct PopularRow { /// first. Popular is the top of it, the directory is all of it, and it is all that /// `subscribe_popular` will subscribe you to. An OPML or a Patreon creator is listed as the /// feeds inside it and never itself: both lists are for finding a show. -async fn popular(state: &WebState, user_id: i64) -> Result> { +/// The catalogue as others may see it. `everything` is the Directory: every feed, with or +/// without subscribers, so the ones an admin listed show before anyone takes them. Without it, +/// Popular: only what people subscribe to. +async fn popular(state: &WebState, user_id: i64, everything: bool) -> Result> { let db = &state.ctx.db; + // One pass for every feed's title, artwork and category, not three queries a feed. + let mut listed = db.feed_list(user_id, None).await?; let mine: std::collections::HashSet = db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect(); let counts = db.subscriber_counts().await?; @@ -886,14 +891,14 @@ async fn popular(state: &WebState, user_id: i64) -> Result> { let n = counts.get(&s.id).copied().unwrap_or(0); // A feed inside an OPML that looks private is as private as the OPML. let folder = s.cfg.group.as_deref().and_then(|g| by_id.get(g)); - if n == 0 + if (n == 0 && !everything) || is_folder.contains(s.id.as_str()) || looks_private(&s.cfg) || folder.is_some_and(|f| looks_private(f)) { continue; } - let sum = db.feed_summary(&s.id).await?; + let sum = listed.remove(&s.id).map(|l| l.summary).unwrap_or_default(); let subscribed = mine.contains(&s.id); out.push(PopularRow { id: s.id.clone(), @@ -914,7 +919,7 @@ async fn get_popular( State(state): State, user: crate::db::User, ) -> Result>, ApiError> { - let mut rows = popular(&state, user.id).await?; + let mut rows = popular(&state, user.id, false).await?; rows.truncate(10); Ok(Json(rows)) } @@ -924,7 +929,7 @@ async fn get_directory( State(state): State, user: crate::db::User, ) -> Result>, ApiError> { - let mut rows = popular(&state, user.id).await?; + let mut rows = popular(&state, user.id, true).await?; rows.sort_by_key(sort_name); Ok(Json(rows)) } @@ -940,10 +945,12 @@ async fn subscribe_popular( user: crate::db::User, Path(id): Path, ) -> Result, ApiError> { - if !popular(&state, user.id).await?.iter().any(|p| p.id == id) { + if !popular(&state, user.id, true).await?.iter().any(|p| p.id == id) { return Err(ApiError::bad_request(format!("{id:?} is not in the directory"))); } state.ctx.db.subscribe(user.id, &id).await?; + // A listed feed nobody took was checked once a day at most: read it now, not in an hour. + scan_soon(&state, Some(id.clone())).await; Ok(Json(serde_json::json!({ "id": id }))) } @@ -1073,6 +1080,7 @@ mod tests { password: None, password_env: None, category: None, + listed: false, }; assert!(!looks_private(&f("https://feeds.twit.tv/twit.xml"))); assert!(!looks_private(&f("https://example.com/rss?format=mp3"))); @@ -1119,7 +1127,7 @@ mod tests { crate::config::Feed { url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false, auto_download: true, max_new_per_check: None, username: None, - password: None, password_env: None, category: None, + password: None, password_env: None, category: None, listed: false, } } @@ -1505,8 +1513,12 @@ async fn remove_feed( } // Nobody is left: the feed stops being scanned. Its files and history stay, so if - // someone subscribes again they do not pull the back catalogue a second time. + // someone subscribes again they do not pull the back catalogue a second time. A feed an + // admin listed stays in the Directory, for the next person. let mut cfg = (*state.ctx.cfg()).clone(); + if cfg.feeds.get(&id).is_some_and(|f| f.listed) { + return Ok(StatusCode::NO_CONTENT); + } if cfg.feeds.remove(&id).is_none() { // A derived feed: forget it here, though the OPML will list it again on the next // read unless you unsubscribe from the OPML itself.