List feeds in the Directory before anyone subscribes, and clear out dead ones (#107)

The Directory showed a catalogue feed only once someone subscribed, and a feed left the
catalogue with its last subscriber, so nothing could be put there for others to find.

- A feed has a listed flag, set by ipx add --list (with --category for the Directory's chip).
  The web page keeps a listed feed in the catalogue when its last subscriber leaves.
- The Directory lists every catalogue feed; Popular still only what people subscribe to.
  popular() reads titles, artwork and categories through Db::feed_list, not three queries a
  feed. Subscribing from the Directory scans the feed at once.
- A feed nobody subscribes to is checked once a day at most.
- clean_directory, in the sweep before each scan, removes from the catalogue and the database
  a feed nobody subscribes to, with no file on disk and not from an OPML, that has failed for
  30 days or published nothing in a year. Run against production first: it removes nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/config.rs

@@ -163,6 +163,10 @@ pub struct Feed {
     /// Preferred over `password`: name of an env var holding the password.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub password_env: Option<String>,
+    /// Put in the Directory by an admin (`ipx add --list`): listed with no subscribers, and kept
+    /// in the catalogue when the last one leaves, where anyone else's feed is dropped.
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub listed: bool,
 }
 
 fn yes() -> bool {
@@ -584,7 +588,7 @@ mod tests {
         taken.insert("the-daily".to_string(), Feed {
             url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
             auto_download: true, max_new_per_check: None, username: None,
-            password: None, password_env: None, category: None,
+            password: None, password_env: None, category: None, listed: false,
         });
         assert_eq!(unique_slug("The Daily", &taken), "the-daily-2");
     }
@@ -605,6 +609,7 @@ mod tests {
             password: Some("literal".into()),
             password_env: None,
             category: None,
+            listed: false,
         };
         assert_eq!(f.password().as_deref(), Some("literal"));
 

src/db.rs

@@ -778,13 +778,47 @@ impl Db {
             .map(|r| Ok(r.try_get("", "id")?))
             .collect::<Result<_>>()?;
         for id in &gone {
-            for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
-                self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.clone().into()]).await?;
-            }
+            self.forget_rows(id).await?;
         }
         Ok(gone)
     }
 
+    /// Everything stored about a feed but its row: items, file rows, read state, block list.
+    async fn forget_rows(&self, id: &str) -> Result<()> {
+        for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
+            self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.into()]).await?;
+        }
+        Ok(())
+    }
+
+    /// Forgets a feed and everything stored about it.
+    pub async fn forget_feed(&self, id: &str) -> Result<()> {
+        self.forget_rows(id).await?;
+        self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?;
+        Ok(())
+    }
+
+    /// Feeds nobody subscribes to that are dead, failing since before `dead_before`, or quiet,
+    /// their newest item from before `quiet_before`: what keeps the Directory to feeds worth
+    /// taking. One with a file on disk stays, and so does one an OPML subscription lists, which
+    /// comes back with the OPML anyway. A feed with no items yet is not quiet, only new.
+    pub async fn stale_unsubscribed(&self, dead_before: i64, quiet_before: i64) -> Result<Vec<String>> {
+        self.rows(
+            "SELECT f.id FROM feeds f
+              WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id)
+                AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = f.id AND x.path IS NOT NULL)
+                AND f.group_id IS NULL AND NOT coalesce(f.managed, false)
+                AND ((f.error_since IS NOT NULL AND f.error_since < $1)
+                     OR (SELECT max(coalesce(e.published, e.first_seen)) FROM entries e
+                          WHERE e.feed_id = f.id) < $2)",
+            vec![dead_before.into(), quiet_before.into()],
+        )
+        .await?
+        .iter()
+        .map(|r| Ok(r.try_get("", "id")?))
+        .collect()
+    }
+
     pub async fn prune_entries(&self, older_than: i64) -> Result<usize> {
         let n = self
             .exec(
@@ -1998,6 +2032,32 @@ mod tests {
         assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
     }
 
+    #[tokio::test]
+    async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() {
+        let db = Db::memory().await.unwrap();
+        db.exec_for_test(
+            "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
+             INSERT INTO feeds (id, url, error_since, managed, group_id) VALUES
+                ('dead','a',100,false,null),('quiet','b',null,false,null),('news','c',null,false,null),
+                ('new','d',null,false,null),('wanted','e',100,false,null),('kept','f',100,false,null),
+                ('child','g',100,true,'opml'),('lately','h',900,false,null);
+             INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted');
+             INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
+                ('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100);
+             INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');",
+        ).await
+        .unwrap();
+        // Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100.
+        // news published lately, new has no items yet, wanted has a subscriber, kept a file,
+        // child comes from an OPML, lately began failing after the cutoff.
+        let mut stale = db.stale_unsubscribed(500, 500).await.unwrap();
+        stale.sort();
+        assert_eq!(stale, ["dead", "quiet"]);
+        db.forget_feed("dead").await.unwrap();
+        assert_eq!(db.feed_summary("dead").await.unwrap().entries, 0);
+        assert!(!db.feed_urls().await.unwrap().iter().any(|(id, _)| id == "dead"));
+    }
+
     #[tokio::test]
     async fn a_failing_feed_nobody_subscribes_to_is_forgotten() {
         let db = Db::memory().await.unwrap();

src/download.rs

@@ -406,7 +406,7 @@ mod tests {
         let mut f = crate::config::Feed {
             url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
             schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
-            max_new_per_check: None, username: None, password: None, password_env: None, category: None,
+            max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false,
         };
         assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");
 

src/main.rs

@@ -63,6 +63,12 @@ enum Command {
         /// Only take enclosures matching these keywords
         #[arg(long, value_delimiter = ',')]
         keywords: Vec<String>,
+        /// The Directory's category, for a feed that names none of its own (News, Technology)
+        #[arg(long)]
+        category: Option<String>,
+        /// Put it in the Directory for anyone to subscribe to, and keep it there when nobody does
+        #[arg(long)]
+        list: bool,
     },
     /// Unsubscribe. Downloads and history are left alone.
     Rm { feed: String },
@@ -259,8 +265,8 @@ async fn main() -> Result<()> {
     let result = match cli.command {
         Command::List => list(&ctx).await,
         Command::Daemon { web } => daemon(ctx, config_path, web, events).await,
-        Command::Add { url, folder, keywords } => {
-            add(&ctx, &url, folder, keywords).await
+        Command::Add { url, folder, keywords, category, list } => {
+            add(&ctx, &url, folder, keywords, category, list).await
         }
         Command::Rm { feed } => rm(&ctx, &feed).await,
         Command::User { cmd } => user_cmd(&ctx, cmd).await,
@@ -673,6 +679,8 @@ async fn add(
     url: &str,
     folder: Option<String>,
     keywords: Vec<String>,
+    category: Option<String>,
+    list: bool,
 ) -> Result<()> {
     let mut cfg = (*ctx.cfg()).clone();
     let url = &feed::find_feed(&ctx.client, &feed::expand_input(url)).await?;
@@ -681,8 +689,12 @@ async fn add(
         anyhow::bail!("already subscribed as {:?}", existing.id);
     }
     let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
+    if let Some(f) = cfg.feeds.get_mut(&id) {
+        f.category = category;
+        f.listed = list;
+    }
     ctx.store_cfg(cfg).await?;
-    println!("added {id}");
+    println!("added {id}{}", if list { ", listed in the Directory" } else { "" });
     Ok(())
 }
 
@@ -709,6 +721,7 @@ pub async fn add_one(
         password: None,
         password_env: None,
         category: None,
+        listed: false,
     };
 
     let title = match feed::fetch(&ctx.client, &probe, None, None).await {
@@ -835,6 +848,7 @@ pub async fn subscribe_opml(
                         password: None,
                         password_env: None,
                         category: None,
+                        listed: false,
                     },
                 );
                 grew = true;
@@ -947,12 +961,38 @@ async fn list(ctx: &Ctx) -> Result<()> {
     Ok(())
 }
 
+/// Removes feeds nobody subscribes to that are dead (failing for 30 days) or quiet (nothing new
+/// in a year), from the catalogue and the database, so the Directory lists feeds worth taking.
+/// A feed listed in it with no subscribers stays for as long as it works and publishes.
+async fn clean_directory(ctx: &Ctx) -> Result<()> {
+    const DEAD: i64 = 30 * 86_400;
+    const QUIET: i64 = 365 * 86_400;
+    let now = db::now();
+    let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?;
+    if stale.is_empty() {
+        return Ok(());
+    }
+    let mut cfg = (*ctx.cfg()).clone();
+    if stale.iter().fold(false, |any, id| cfg.feeds.remove(id).is_some() || any) {
+        ctx.store_cfg(cfg).await?;
+    }
+    for id in &stale {
+        retire_group(ctx, id).await?;
+        ctx.db.forget_feed(id).await?;
+        tracing::info!(feed = id, "removed a feed nobody subscribes to that is dead or has published nothing in a year");
+    }
+    Ok(())
+}
+
 /// `standalone` false means this is the sweep that runs before a scan: it reports what it
 /// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch`
 /// would stop reading before the scan had even started.
 #[tracing::instrument(name = "reap", skip_all, fields(dry_run = dry_run))]
 async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
     let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?;
+    if !dry_run {
+        clean_directory(ctx).await?;
+    }
     for c in r.aged_out.iter().chain(r.over_quota.iter()) {
         ctx.out.emit(Event::Reaped {
             path: c.path.clone(),
@@ -979,6 +1019,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
         anyhow::bail!("no feed with id {id:?}");
     }
 
+    let subscribed = ctx.db.subscriber_counts().await?;
     let mut scanned = 0;
     let mut fresh: Vec<String> = vec![];
     let in_scope = |s: &Sub| {
@@ -996,7 +1037,11 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
         }
 
         if !force && let Some(last) = state.last_checked {
-            let at = last + due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last) as i64;
+            // A feed nobody subscribes to, one listed in the Directory, is read once a day: enough
+            // to keep its entry current, without fetching it hourly for no one.
+            let floor = if subscribed.contains_key(id) { 0 } else { 86_400 };
+            let wait = due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last).max(floor);
+            let at = last + wait as i64;
             if at > db::now() {
                 ctx.out.emit(Event::FeedSkip {
                     feed: id.clone(),
@@ -1156,6 +1201,7 @@ pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
                 password: parent.and_then(|p| p.password.clone()),
                 password_env: parent.and_then(|p| p.password_env.clone()),
                 category: None,
+                listed: false,
             },
             managed: true,
         });
@@ -1969,6 +2015,7 @@ mod tests {
             password: None,
             password_env: None,
             category: None,
+            listed: false,
         }
     }
 

src/web.rs

@@ -870,8 +870,13 @@ struct PopularRow {
 /// first. Popular is the top of it, the directory is all of it, and it is all that
 /// `subscribe_popular` will subscribe you to. An OPML or a Patreon creator is listed as the
 /// feeds inside it and never itself: both lists are for finding a show.
-async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> {
+/// The catalogue as others may see it. `everything` is the Directory: every feed, with or
+/// without subscribers, so the ones an admin listed show before anyone takes them. Without it,
+/// Popular: only what people subscribe to.
+async fn popular(state: &WebState, user_id: i64, everything: bool) -> Result<Vec<PopularRow>> {
     let db = &state.ctx.db;
+    // One pass for every feed's title, artwork and category, not three queries a feed.
+    let mut listed = db.feed_list(user_id, None).await?;
     let mine: std::collections::HashSet<String> =
         db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect();
     let counts = db.subscriber_counts().await?;
@@ -886,14 +891,14 @@ async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> {
         let n = counts.get(&s.id).copied().unwrap_or(0);
         // A feed inside an OPML that looks private is as private as the OPML.
         let folder = s.cfg.group.as_deref().and_then(|g| by_id.get(g));
-        if n == 0
+        if (n == 0 && !everything)
             || is_folder.contains(s.id.as_str())
             || looks_private(&s.cfg)
             || folder.is_some_and(|f| looks_private(f))
         {
             continue;
         }
-        let sum = db.feed_summary(&s.id).await?;
+        let sum = listed.remove(&s.id).map(|l| l.summary).unwrap_or_default();
         let subscribed = mine.contains(&s.id);
         out.push(PopularRow {
             id: s.id.clone(),
@@ -914,7 +919,7 @@ async fn get_popular(
     State(state): State<WebState>,
     user: crate::db::User,
 ) -> Result<Json<Vec<PopularRow>>, ApiError> {
-    let mut rows = popular(&state, user.id).await?;
+    let mut rows = popular(&state, user.id, false).await?;
     rows.truncate(10);
     Ok(Json(rows))
 }
@@ -924,7 +929,7 @@ async fn get_directory(
     State(state): State<WebState>,
     user: crate::db::User,
 ) -> Result<Json<Vec<PopularRow>>, ApiError> {
-    let mut rows = popular(&state, user.id).await?;
+    let mut rows = popular(&state, user.id, true).await?;
     rows.sort_by_key(sort_name);
     Ok(Json(rows))
 }
@@ -940,10 +945,12 @@ async fn subscribe_popular(
     user: crate::db::User,
     Path(id): Path<String>,
 ) -> Result<Json<serde_json::Value>, ApiError> {
-    if !popular(&state, user.id).await?.iter().any(|p| p.id == id) {
+    if !popular(&state, user.id, true).await?.iter().any(|p| p.id == id) {
         return Err(ApiError::bad_request(format!("{id:?} is not in the directory")));
     }
     state.ctx.db.subscribe(user.id, &id).await?;
+    // A listed feed nobody took was checked once a day at most: read it now, not in an hour.
+    scan_soon(&state, Some(id.clone())).await;
     Ok(Json(serde_json::json!({ "id": id })))
 }
 
@@ -1073,6 +1080,7 @@ mod tests {
             password: None,
             password_env: None,
             category: None,
+            listed: false,
         };
         assert!(!looks_private(&f("https://feeds.twit.tv/twit.xml")));
         assert!(!looks_private(&f("https://example.com/rss?format=mp3")));
@@ -1119,7 +1127,7 @@ mod tests {
         crate::config::Feed {
             url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
             auto_download: true, max_new_per_check: None, username: None,
-            password: None, password_env: None, category: None,
+            password: None, password_env: None, category: None, listed: false,
         }
     }
 
@@ -1505,8 +1513,12 @@ async fn remove_feed(
     }
 
     // Nobody is left: the feed stops being scanned. Its files and history stay, so if
-    // someone subscribes again they do not pull the back catalogue a second time.
+    // someone subscribes again they do not pull the back catalogue a second time. A feed an
+    // admin listed stays in the Directory, for the next person.
     let mut cfg = (*state.ctx.cfg()).clone();
+    if cfg.feeds.get(&id).is_some_and(|f| f.listed) {
+        return Ok(StatusCode::NO_CONTENT);
+    }
     if cfg.feeds.remove(&id).is_none() {
         // A derived feed: forget it here, though the OPML will list it again on the next
         // read unless you unsubscribe from the OPML itself.