List feeds in the Directory before anyone subscribes, and clear out dead ones (#107)

The Directory showed a catalogue feed only once someone subscribed, and a feed left the
catalogue with its last subscriber, so nothing could be put there for others to find.

- A feed has a listed flag, set by ipx add --list (with --category for the Directory's chip).
  The web page keeps a listed feed in the catalogue when its last subscriber leaves.
- The Directory lists every catalogue feed; Popular still only what people subscribe to.
  popular() reads titles, artwork and categories through Db::feed_list, not three queries a
  feed. Subscribing from the Directory scans the feed at once.
- A feed nobody subscribes to is checked once a day at most.
- clean_directory, in the sweep before each scan, removes from the catalogue and the database
  a feed nobody subscribes to, with no file on disk and not from an OPML, that has failed for
  30 days or published nothing in a year. Run against production first: it removes nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main.rs

@@ -63,6 +63,12 @@ enum Command {
         /// Only take enclosures matching these keywords
         #[arg(long, value_delimiter = ',')]
         keywords: Vec<String>,
+        /// The Directory's category, for a feed that names none of its own (News, Technology)
+        #[arg(long)]
+        category: Option<String>,
+        /// Put it in the Directory for anyone to subscribe to, and keep it there when nobody does
+        #[arg(long)]
+        list: bool,
     },
     /// Unsubscribe. Downloads and history are left alone.
     Rm { feed: String },
@@ -259,8 +265,8 @@ async fn main() -> Result<()> {
     let result = match cli.command {
         Command::List => list(&ctx).await,
         Command::Daemon { web } => daemon(ctx, config_path, web, events).await,
-        Command::Add { url, folder, keywords } => {
-            add(&ctx, &url, folder, keywords).await
+        Command::Add { url, folder, keywords, category, list } => {
+            add(&ctx, &url, folder, keywords, category, list).await
         }
         Command::Rm { feed } => rm(&ctx, &feed).await,
         Command::User { cmd } => user_cmd(&ctx, cmd).await,
@@ -673,6 +679,8 @@ async fn add(
     url: &str,
     folder: Option<String>,
     keywords: Vec<String>,
+    category: Option<String>,
+    list: bool,
 ) -> Result<()> {
     let mut cfg = (*ctx.cfg()).clone();
     let url = &feed::find_feed(&ctx.client, &feed::expand_input(url)).await?;
@@ -681,8 +689,12 @@ async fn add(
         anyhow::bail!("already subscribed as {:?}", existing.id);
     }
     let id = add_one(ctx, &mut cfg, url, folder, keywords).await?;
+    if let Some(f) = cfg.feeds.get_mut(&id) {
+        f.category = category;
+        f.listed = list;
+    }
     ctx.store_cfg(cfg).await?;
-    println!("added {id}");
+    println!("added {id}{}", if list { ", listed in the Directory" } else { "" });
     Ok(())
 }
 
@@ -709,6 +721,7 @@ pub async fn add_one(
         password: None,
         password_env: None,
         category: None,
+        listed: false,
     };
 
     let title = match feed::fetch(&ctx.client, &probe, None, None).await {
@@ -835,6 +848,7 @@ pub async fn subscribe_opml(
                         password: None,
                         password_env: None,
                         category: None,
+                        listed: false,
                     },
                 );
                 grew = true;
@@ -947,12 +961,38 @@ async fn list(ctx: &Ctx) -> Result<()> {
     Ok(())
 }
 
+/// Removes feeds nobody subscribes to that are dead (failing for 30 days) or quiet (nothing new
+/// in a year), from the catalogue and the database, so the Directory lists feeds worth taking.
+/// A feed listed in it with no subscribers stays for as long as it works and publishes.
+async fn clean_directory(ctx: &Ctx) -> Result<()> {
+    const DEAD: i64 = 30 * 86_400;
+    const QUIET: i64 = 365 * 86_400;
+    let now = db::now();
+    let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?;
+    if stale.is_empty() {
+        return Ok(());
+    }
+    let mut cfg = (*ctx.cfg()).clone();
+    if stale.iter().fold(false, |any, id| cfg.feeds.remove(id).is_some() || any) {
+        ctx.store_cfg(cfg).await?;
+    }
+    for id in &stale {
+        retire_group(ctx, id).await?;
+        ctx.db.forget_feed(id).await?;
+        tracing::info!(feed = id, "removed a feed nobody subscribes to that is dead or has published nothing in a year");
+    }
+    Ok(())
+}
+
 /// `standalone` false means this is the sweep that runs before a scan: it reports what it
 /// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch`
 /// would stop reading before the scan had even started.
 #[tracing::instrument(name = "reap", skip_all, fields(dry_run = dry_run))]
 async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
     let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?;
+    if !dry_run {
+        clean_directory(ctx).await?;
+    }
     for c in r.aged_out.iter().chain(r.over_quota.iter()) {
         ctx.out.emit(Event::Reaped {
             path: c.path.clone(),
@@ -979,6 +1019,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
         anyhow::bail!("no feed with id {id:?}");
     }
 
+    let subscribed = ctx.db.subscriber_counts().await?;
     let mut scanned = 0;
     let mut fresh: Vec<String> = vec![];
     let in_scope = |s: &Sub| {
@@ -996,7 +1037,11 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
         }
 
         if !force && let Some(last) = state.last_checked {
-            let at = last + due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last) as i64;
+            // A feed nobody subscribes to, one listed in the Directory, is read once a day: enough
+            // to keep its entry current, without fetching it hourly for no one.
+            let floor = if subscribed.contains_key(id) { 0 } else { 86_400 };
+            let wait = due_after(&cfg, &sub.cfg, state.ttl_mins, state.error_since, last).max(floor);
+            let at = last + wait as i64;
             if at > db::now() {
                 ctx.out.emit(Event::FeedSkip {
                     feed: id.clone(),
@@ -1156,6 +1201,7 @@ pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
                 password: parent.and_then(|p| p.password.clone()),
                 password_env: parent.and_then(|p| p.password_env.clone()),
                 category: None,
+                listed: false,
             },
             managed: true,
         });
@@ -1969,6 +2015,7 @@ mod tests {
             password: None,
             password_env: None,
             category: None,
+            listed: false,
         }
     }