Keep a show from an OPML when the OPML drops it (#280)

A show inside a subscribed OPML went when the OPML stopped listing it
(orphaned only if it had downloads), and went for you when you left the
OPML. Keep, in its settings, is a per-person flag on the subscription:
when the OPML drops a kept show, or nobody subscribes to the OPML any more,
it becomes a catalogue entry outside the OPML, taking the OPML's settings
while it is there, and stays on the lists of those who kept it. Whoever had
it only through the OPML loses it as before. Leaving the OPML yourself
leaves what you kept on your list.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/ipx/CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- A show from an OPML subscription or a Patreon creator can be kept: `kept` on `PATCH /api/feeds/{id}` and in the feed list. When the OPML drops a kept show, or you leave the OPML, it stays on your list as a feed of its own (#280).
 - `/metrics` counts Signals: items judged and fitting, calls to Jev by how they ended, the time they took and the tokens TypeSafe counted, time since the judge's last pass, and keys refused; a failed call to Jev shows as a failed trace (#270, #275).
 - An admin setting for how many days back a new Signal reads (#276).
 - `/metrics` breaks Signals' items, fits and TypeSafe tokens out by person and by whose key paid, so the server's bill can be split (#278).

crates/ipx/src/db.rs

@@ -110,6 +110,10 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
         let sql = "ALTER TABLE users ADD COLUMN typesafe_key TEXT";
         orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
     }
+    if orm.execute_unprepared("SELECT kept FROM subscriptions LIMIT 0").await.is_err() {
+        let sql = "ALTER TABLE subscriptions ADD COLUMN kept BOOLEAN NOT NULL DEFAULT false";
+        orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
+    }
     if orm.execute_unprepared("SELECT description FROM signals LIMIT 0").await.is_err() {
         // Every verdict so far answered the old question, any one keyword on its own, which put
         // a game's remaster in AI Development (#259): all of them are asked again.
@@ -1794,6 +1798,40 @@ impl Db {
             .collect())
     }
 
+    /// The feeds from a list this person keeps should the list drop them (#280).
+    pub async fn kept_feeds(&self, user_id: i64) -> Result<std::collections::HashSet<String>> {
+        Ok(subscriptions::Entity::find()
+            .filter(subscriptions::Column::UserId.eq(user_id))
+            .filter(subscriptions::Column::Kept.eq(true))
+            .all(&self.orm)
+            .await?
+            .into_iter()
+            .map(|s| s.feed_id)
+            .collect())
+    }
+
+    /// False when they do not subscribe to it, as `set_pinned`.
+    pub async fn set_kept(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
+        let r = subscriptions::Entity::update_many()
+            .col_expr(subscriptions::Column::Kept, Expr::val(on).into())
+            .filter(subscriptions::Column::UserId.eq(user_id))
+            .filter(subscriptions::Column::FeedId.eq(feed_id))
+            .exec(&self.orm)
+            .await?;
+        Ok(r.rows_affected > 0)
+    }
+
+    /// Who subscribes to a feed, and whether each keeps it (#280).
+    pub async fn subscribers_of(&self, feed_id: &str) -> Result<Vec<(i64, bool)>> {
+        Ok(subscriptions::Entity::find()
+            .filter(subscriptions::Column::FeedId.eq(feed_id))
+            .all(&self.orm)
+            .await?
+            .into_iter()
+            .map(|s| (s.user_id, s.kept))
+            .collect())
+    }
+
     /// False when they do not subscribe to it, since there is then no row in their list to pin.
     /// Whether a feed or an entry names `url` as its artwork: the only addresses /api/art will
     /// fetch, so the web server cannot be pointed at anything else.

crates/ipx/src/entity.rs

@@ -216,6 +216,10 @@ pub mod subscriptions {
         /// Pinned to the top of this person's feed list, a feed inside a folder included.
         #[sea_orm(default_value = false)]
         pub pinned: bool,
+        /// A feed from an OPML or Patreon list this person keeps when the list drops it, or when
+        /// they leave the list: it becomes a feed of its own instead of going (#280).
+        #[sea_orm(default_value = false)]
+        pub kept: bool,
     }
 
     owned_by_user!();

crates/ipx/src/main.rs

@@ -1451,7 +1451,9 @@ pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
 /// lists are left alone.
 pub async fn leave(ctx: &Ctx, user_id: i64, id: &str) -> Result<()> {
     ctx.db.unsubscribe(user_id, id).await?;
-    for child in subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id)) {
+    // What they kept stays on their list, a feed of its own once nobody reads the list (#280).
+    let kept = ctx.db.kept_feeds(user_id).await?;
+    for child in subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id)) {
         ctx.db.unsubscribe(user_id, &child.id).await?;
     }
     if ctx.db.subscriber_counts().await?.contains_key(id) {
@@ -1504,6 +1506,8 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
             // Scanned from its config entry and still read. Dropped as derived, its stored
             // entries would go with it: davewiner's 11 were promoted without being unmanaged.
             ctx.db.unmanage(&m.id).await?;
+        } else if promote_kept(ctx, &m).await? {
+            tracing::info!(feed = %m.id, "its list is gone but someone kept it; a feed of its own now");
         } else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
             ctx.db.set_orphaned(&m.id, true).await?;
         } else {
@@ -1513,6 +1517,46 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
     Ok(())
 }
 
+/// Makes a feed from an OPML or Patreon list a feed of its own, for those who kept it (#280): a
+/// catalogue entry in no group, which the list no longer adds or takes away. Whoever had it only
+/// through the list, without keeping it, loses it, as they would have when the list dropped it.
+/// False, changing nothing, when nobody kept it.
+async fn promote_kept(ctx: &Ctx, m: &db::Managed) -> Result<bool> {
+    let subs = ctx.db.subscribers_of(&m.id).await?;
+    if !subs.iter().any(|(_, kept)| *kept) {
+        return Ok(false);
+    }
+    let mut cfg = ctx.edit_cfg().await;
+    // The list's own settings carry over while it is there; its folder and group do not.
+    let base = cfg.feeds.get(&m.group_id).cloned();
+    cfg.feeds.entry(m.id.clone()).or_insert_with(|| config::Feed {
+        url: m.url.clone(),
+        folder: None,
+        group: None,
+        media_types: base.as_ref().and_then(|p| p.media_types.clone()),
+        schedule: base.as_ref().and_then(|p| p.schedule.clone()),
+        keywords: base.as_ref().map(|p| p.keywords.clone()).unwrap_or_default(),
+        allow_explicit: base.as_ref().is_some_and(|p| p.allow_explicit),
+        auto_download: base.as_ref().is_none_or(|p| p.auto_download),
+        max_new_per_check: base.as_ref().and_then(|p| p.max_new_per_check),
+        username: base.as_ref().and_then(|p| p.username.clone()),
+        password: base.as_ref().and_then(|p| p.password.clone()),
+        password_env: base.as_ref().and_then(|p| p.password_env.clone()),
+        category: None,
+        listed: false,
+        private: base.as_ref().is_some_and(|p| p.private),
+    });
+    ctx.store_cfg(cfg).await?;
+    ctx.db.unmanage(&m.id).await?;
+    ctx.db.set_orphaned(&m.id, false).await?;
+    for (user, kept) in subs {
+        if !kept {
+            ctx.db.unsubscribe(user, &m.id).await?;
+        }
+    }
+    Ok(true)
+}
+
 /// Retires every group whose parent is gone from config, and returns how many derived rows that
 /// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind:
 /// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were
@@ -2066,6 +2110,11 @@ async fn sync_group(
         if listed.iter().any(|(_, u)| u == &m.url) {
             continue;
         }
+        if promote_kept(ctx, m).await? {
+            kept += 1;
+            tracing::info!(feed = %m.id, "dropped from the OPML but someone kept it; a feed of its own now");
+            continue;
+        }
         if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
             // Never orphan a downloaded file: keep the feed and say why in the UI.
             ctx.db.set_orphaned(&m.id, true).await?;
@@ -2963,6 +3012,43 @@ mod tests {
         assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned");
     }
 
+    #[tokio::test]
+    async fn a_kept_feed_outlives_its_opml_for_whoever_kept_it() {
+        let mut cfg = config::Config::default();
+        cfg.feeds.insert("opml".into(), feed());
+        let ctx = Arc::new(test_ctx(cfg).await);
+        let (me, you) = (ctx.db.create_user("me", None, false).await.unwrap(), ctx.db.create_user("you", None, false).await.unwrap());
+        for id in ["opml", "a", "b"] {
+            if id != "opml" {
+                ctx.db.upsert_managed(id, &format!("http://x/{id}.xml"), id, "opml").await.unwrap();
+            }
+            for u in [me, you] {
+                ctx.db.subscribe(u, id).await.unwrap();
+            }
+        }
+        assert!(ctx.db.set_kept(me, "a", true).await.unwrap());
+        assert!(ctx.db.set_kept(me, "b", true).await.unwrap());
+
+        // The OPML drops a: mine to keep, a feed of its own; yours, unkept, goes.
+        let parent = ctx.cfg().feeds["opml"].clone();
+        sync_group(&ctx, "opml", &parent, &[("b".into(), "http://x/b.xml".into())]).await.unwrap();
+        let a = ctx.cfg().feeds.get("a").cloned().expect("a is in the catalogue");
+        assert_eq!((a.url.as_str(), a.group), ("http://x/a.xml", None));
+        assert!(!ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "a"));
+        assert!(ctx.db.subscription(me, "a").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "a").await.unwrap().is_none());
+
+        // Leaving the OPML takes what I did not keep and leaves b, still the OPML's while you read it.
+        leave(&ctx, me, "opml").await.unwrap();
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "b"));
+        // Once nobody reads it, b is mine alone, and yours goes with the OPML.
+        leave(&ctx, you, "opml").await.unwrap();
+        assert!(ctx.cfg().feeds.get("b").is_some_and(|f| f.group.is_none()));
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "b").await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() {
         // davewiner: the OPML left config before retire_group existed, and 11 of its feeds

crates/ipx/src/web.rs

@@ -869,6 +869,8 @@ struct FeedRow {
     subscribers: i64,
     /// Pinned to the top of your list.
     pinned: bool,
+    /// From an OPML, and yours to keep should it drop the feed (#280).
+    kept: bool,
     /// Marked private: kept out of the directory (#205).
     private: bool,
     /// Private whatever it is marked, for a login or an address with a key in it.
@@ -909,6 +911,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
         .collect();
     let counts = state.ctx.db.subscriber_counts().instrument(step("subscriber_counts")).await?;
     let pinned = state.ctx.db.pinned_feeds(user.id).instrument(step("pinned")).await?;
+    let kept = state.ctx.db.kept_feeds(user.id).instrument(step("kept")).await?;
     let prefs = state.ctx.db.deadair_prefs(Some(user.id)).await?;
     let deadair: std::collections::HashMap<&str, bool> = prefs.iter().map(|(_, f, w, _)| (f.as_str(), *w)).collect();
     let pauses: std::collections::HashMap<&str, i64> =
@@ -988,6 +991,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
             unread,
             subscribers: counts.get(id).copied().unwrap_or(0),
             pinned: pinned.contains(id),
+            kept: kept.contains(id),
         });
     }
     Ok(out)
@@ -1638,6 +1642,8 @@ struct FeedPatch {
     #[serde(default, deserialize_with = "double_option")]
     max_new_per_check: Option<Option<usize>>,
     pinned: Option<bool>,
+    /// Kept should its OPML drop it (#280).
+    kept: Option<bool>,
     /// True plays the feed shortened by DeadAir, false keeps it out (of its group's too), null
     /// follows its group again (#171).
     #[serde(default, deserialize_with = "double_option")]
@@ -1684,6 +1690,11 @@ async fn patch_feed(
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
+    if let Some(on) = body.kept
+        && !state.ctx.db.set_kept(user.id, &id, on).await?
+    {
+        return Err(ApiError::not_found("you do not subscribe to that feed"));
+    }
     if let Some(p) = body.deadair_pause
         && !crate::deadair::PAUSES.contains(&p)
     {