Keep a show from an OPML when the OPML drops it (#280)

A show inside a subscribed OPML went when the OPML stopped listing it
(orphaned only if it had downloads), and went for you when you left the
OPML. Keep, in its settings, is a per-person flag on the subscription:
when the OPML drops a kept show, or nobody subscribes to the OPML any more,
it becomes a catalogue entry outside the OPML, taking the OPML's settings
while it is there, and stays on the lists of those who kept it. Whoever had
it only through the OPML loses it as before. Leaving the OPML yourself
leaves what you kept on your list.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/ipx/CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- A show from an OPML subscription or a Patreon creator can be kept: `kept` on `PATCH /api/feeds/{id}` and in the feed list. When the OPML drops a kept show, or you leave the OPML, it stays on your list as a feed of its own (#280).
 - `/metrics` counts Signals: items judged and fitting, calls to Jev by how they ended, the time they took and the tokens TypeSafe counted, time since the judge's last pass, and keys refused; a failed call to Jev shows as a failed trace (#270, #275).
 - An admin setting for how many days back a new Signal reads (#276).
 - `/metrics` breaks Signals' items, fits and TypeSafe tokens out by person and by whose key paid, so the server's bill can be split (#278).

crates/ipx/src/db.rs

@@ -110,6 +110,10 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
         let sql = "ALTER TABLE users ADD COLUMN typesafe_key TEXT";
         orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
     }
+    if orm.execute_unprepared("SELECT kept FROM subscriptions LIMIT 0").await.is_err() {
+        let sql = "ALTER TABLE subscriptions ADD COLUMN kept BOOLEAN NOT NULL DEFAULT false";
+        orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
+    }
     if orm.execute_unprepared("SELECT description FROM signals LIMIT 0").await.is_err() {
         // Every verdict so far answered the old question, any one keyword on its own, which put
         // a game's remaster in AI Development (#259): all of them are asked again.
@@ -1794,6 +1798,40 @@ impl Db {
             .collect())
     }
 
+    /// The feeds from a list this person keeps should the list drop them (#280).
+    pub async fn kept_feeds(&self, user_id: i64) -> Result<std::collections::HashSet<String>> {
+        Ok(subscriptions::Entity::find()
+            .filter(subscriptions::Column::UserId.eq(user_id))
+            .filter(subscriptions::Column::Kept.eq(true))
+            .all(&self.orm)
+            .await?
+            .into_iter()
+            .map(|s| s.feed_id)
+            .collect())
+    }
+
+    /// False when they do not subscribe to it, as `set_pinned`.
+    pub async fn set_kept(&self, user_id: i64, feed_id: &str, on: bool) -> Result<bool> {
+        let r = subscriptions::Entity::update_many()
+            .col_expr(subscriptions::Column::Kept, Expr::val(on).into())
+            .filter(subscriptions::Column::UserId.eq(user_id))
+            .filter(subscriptions::Column::FeedId.eq(feed_id))
+            .exec(&self.orm)
+            .await?;
+        Ok(r.rows_affected > 0)
+    }
+
+    /// Who subscribes to a feed, and whether each keeps it (#280).
+    pub async fn subscribers_of(&self, feed_id: &str) -> Result<Vec<(i64, bool)>> {
+        Ok(subscriptions::Entity::find()
+            .filter(subscriptions::Column::FeedId.eq(feed_id))
+            .all(&self.orm)
+            .await?
+            .into_iter()
+            .map(|s| (s.user_id, s.kept))
+            .collect())
+    }
+
     /// False when they do not subscribe to it, since there is then no row in their list to pin.
     /// Whether a feed or an entry names `url` as its artwork: the only addresses /api/art will
     /// fetch, so the web server cannot be pointed at anything else.

crates/ipx/src/entity.rs

@@ -216,6 +216,10 @@ pub mod subscriptions {
         /// Pinned to the top of this person's feed list, a feed inside a folder included.
         #[sea_orm(default_value = false)]
         pub pinned: bool,
+        /// A feed from an OPML or Patreon list this person keeps when the list drops it, or when
+        /// they leave the list: it becomes a feed of its own instead of going (#280).
+        #[sea_orm(default_value = false)]
+        pub kept: bool,
     }
 
     owned_by_user!();

crates/ipx/src/main.rs

@@ -1451,7 +1451,9 @@ pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
 /// lists are left alone.
 pub async fn leave(ctx: &Ctx, user_id: i64, id: &str) -> Result<()> {
     ctx.db.unsubscribe(user_id, id).await?;
-    for child in subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id)) {
+    // What they kept stays on their list, a feed of its own once nobody reads the list (#280).
+    let kept = ctx.db.kept_feeds(user_id).await?;
+    for child in subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id)) {
         ctx.db.unsubscribe(user_id, &child.id).await?;
     }
     if ctx.db.subscriber_counts().await?.contains_key(id) {
@@ -1504,6 +1506,8 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
             // Scanned from its config entry and still read. Dropped as derived, its stored
             // entries would go with it: davewiner's 11 were promoted without being unmanaged.
             ctx.db.unmanage(&m.id).await?;
+        } else if promote_kept(ctx, &m).await? {
+            tracing::info!(feed = %m.id, "its list is gone but someone kept it; a feed of its own now");
         } else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
             ctx.db.set_orphaned(&m.id, true).await?;
         } else {
@@ -1513,6 +1517,46 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> {
     Ok(())
 }
 
+/// Makes a feed from an OPML or Patreon list a feed of its own, for those who kept it (#280): a
+/// catalogue entry in no group, which the list no longer adds or takes away. Whoever had it only
+/// through the list, without keeping it, loses it, as they would have when the list dropped it.
+/// False, changing nothing, when nobody kept it.
+async fn promote_kept(ctx: &Ctx, m: &db::Managed) -> Result<bool> {
+    let subs = ctx.db.subscribers_of(&m.id).await?;
+    if !subs.iter().any(|(_, kept)| *kept) {
+        return Ok(false);
+    }
+    let mut cfg = ctx.edit_cfg().await;
+    // The list's own settings carry over while it is there; its folder and group do not.
+    let base = cfg.feeds.get(&m.group_id).cloned();
+    cfg.feeds.entry(m.id.clone()).or_insert_with(|| config::Feed {
+        url: m.url.clone(),
+        folder: None,
+        group: None,
+        media_types: base.as_ref().and_then(|p| p.media_types.clone()),
+        schedule: base.as_ref().and_then(|p| p.schedule.clone()),
+        keywords: base.as_ref().map(|p| p.keywords.clone()).unwrap_or_default(),
+        allow_explicit: base.as_ref().is_some_and(|p| p.allow_explicit),
+        auto_download: base.as_ref().is_none_or(|p| p.auto_download),
+        max_new_per_check: base.as_ref().and_then(|p| p.max_new_per_check),
+        username: base.as_ref().and_then(|p| p.username.clone()),
+        password: base.as_ref().and_then(|p| p.password.clone()),
+        password_env: base.as_ref().and_then(|p| p.password_env.clone()),
+        category: None,
+        listed: false,
+        private: base.as_ref().is_some_and(|p| p.private),
+    });
+    ctx.store_cfg(cfg).await?;
+    ctx.db.unmanage(&m.id).await?;
+    ctx.db.set_orphaned(&m.id, false).await?;
+    for (user, kept) in subs {
+        if !kept {
+            ctx.db.unsubscribe(user, &m.id).await?;
+        }
+    }
+    Ok(true)
+}
+
 /// Retires every group whose parent is gone from config, and returns how many derived rows that
 /// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind:
 /// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were
@@ -2066,6 +2110,11 @@ async fn sync_group(
         if listed.iter().any(|(_, u)| u == &m.url) {
             continue;
         }
+        if promote_kept(ctx, m).await? {
+            kept += 1;
+            tracing::info!(feed = %m.id, "dropped from the OPML but someone kept it; a feed of its own now");
+            continue;
+        }
         if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 {
             // Never orphan a downloaded file: keep the feed and say why in the UI.
             ctx.db.set_orphaned(&m.id, true).await?;
@@ -2963,6 +3012,43 @@ mod tests {
         assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned");
     }
 
+    #[tokio::test]
+    async fn a_kept_feed_outlives_its_opml_for_whoever_kept_it() {
+        let mut cfg = config::Config::default();
+        cfg.feeds.insert("opml".into(), feed());
+        let ctx = Arc::new(test_ctx(cfg).await);
+        let (me, you) = (ctx.db.create_user("me", None, false).await.unwrap(), ctx.db.create_user("you", None, false).await.unwrap());
+        for id in ["opml", "a", "b"] {
+            if id != "opml" {
+                ctx.db.upsert_managed(id, &format!("http://x/{id}.xml"), id, "opml").await.unwrap();
+            }
+            for u in [me, you] {
+                ctx.db.subscribe(u, id).await.unwrap();
+            }
+        }
+        assert!(ctx.db.set_kept(me, "a", true).await.unwrap());
+        assert!(ctx.db.set_kept(me, "b", true).await.unwrap());
+
+        // The OPML drops a: mine to keep, a feed of its own; yours, unkept, goes.
+        let parent = ctx.cfg().feeds["opml"].clone();
+        sync_group(&ctx, "opml", &parent, &[("b".into(), "http://x/b.xml".into())]).await.unwrap();
+        let a = ctx.cfg().feeds.get("a").cloned().expect("a is in the catalogue");
+        assert_eq!((a.url.as_str(), a.group), ("http://x/a.xml", None));
+        assert!(!ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "a"));
+        assert!(ctx.db.subscription(me, "a").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "a").await.unwrap().is_none());
+
+        // Leaving the OPML takes what I did not keep and leaves b, still the OPML's while you read it.
+        leave(&ctx, me, "opml").await.unwrap();
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.managed_feeds().await.unwrap().iter().any(|m| m.id == "b"));
+        // Once nobody reads it, b is mine alone, and yours goes with the OPML.
+        leave(&ctx, you, "opml").await.unwrap();
+        assert!(ctx.cfg().feeds.get("b").is_some_and(|f| f.group.is_none()));
+        assert!(ctx.db.subscription(me, "b").await.unwrap().is_some());
+        assert!(ctx.db.subscription(you, "b").await.unwrap().is_none());
+    }
+
     #[tokio::test]
     async fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() {
         // davewiner: the OPML left config before retire_group existed, and 11 of its feeds

crates/ipx/src/web.rs

@@ -869,6 +869,8 @@ struct FeedRow {
     subscribers: i64,
     /// Pinned to the top of your list.
     pinned: bool,
+    /// From an OPML, and yours to keep should it drop the feed (#280).
+    kept: bool,
     /// Marked private: kept out of the directory (#205).
     private: bool,
     /// Private whatever it is marked, for a login or an address with a key in it.
@@ -909,6 +911,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
         .collect();
     let counts = state.ctx.db.subscriber_counts().instrument(step("subscriber_counts")).await?;
     let pinned = state.ctx.db.pinned_feeds(user.id).instrument(step("pinned")).await?;
+    let kept = state.ctx.db.kept_feeds(user.id).instrument(step("kept")).await?;
     let prefs = state.ctx.db.deadair_prefs(Some(user.id)).await?;
     let deadair: std::collections::HashMap<&str, bool> = prefs.iter().map(|(_, f, w, _)| (f.as_str(), *w)).collect();
     let pauses: std::collections::HashMap<&str, i64> =
@@ -988,6 +991,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
             unread,
             subscribers: counts.get(id).copied().unwrap_or(0),
             pinned: pinned.contains(id),
+            kept: kept.contains(id),
         });
     }
     Ok(out)
@@ -1638,6 +1642,8 @@ struct FeedPatch {
     #[serde(default, deserialize_with = "double_option")]
     max_new_per_check: Option<Option<usize>>,
     pinned: Option<bool>,
+    /// Kept should its OPML drop it (#280).
+    kept: Option<bool>,
     /// True plays the feed shortened by DeadAir, false keeps it out (of its group's too), null
     /// follows its group again (#171).
     #[serde(default, deserialize_with = "double_option")]
@@ -1684,6 +1690,11 @@ async fn patch_feed(
     {
         return Err(ApiError::not_found("you do not subscribe to that feed"));
     }
+    if let Some(on) = body.kept
+        && !state.ctx.db.set_kept(user.id, &id, on).await?
+    {
+        return Err(ApiError::not_found("you do not subscribe to that feed"));
+    }
     if let Some(p) = body.deadair_pause
         && !crate::deadair::PAUSES.contains(&p)
     {

docs/configuration.md

@@ -165,7 +165,11 @@ subscribes to is checked once a day. One that is dead (failing for 30 days) or q
 new in a year), with nobody subscribed and no file on disk, is removed from the catalogue.
 
 Feeds derived from a subscribed OPML are **not** in the catalogue: the OPML is the source of truth
-and they are re-derived on every scan. Editing one in the UI promotes it to a catalogue entry.
+and they are re-derived on every scan. Editing one in the UI promotes it to a catalogue entry. So
+does Keep, in a show's settings, for whoever turns it on (`kept` on their subscription, #280): when
+the OPML stops listing the show, or nobody subscribes to the OPML any more, it becomes a catalogue
+entry outside the OPML instead of going, and stays on the lists of those who kept it. Anyone who
+had it only through the OPML loses it as before. Leaving the OPML yourself keeps what you kept.
 
 ## Environment
 

site/api.mjs

@@ -117,7 +117,7 @@ export const ipx = {
         {
           method: 'PATCH', path: '/api/feeds/{id}', slug: 'patch-feed',
           summary: 'Change your settings for a feed, or (an admin) the feed’s own.',
-          doc: '<p>Send only the fields to change. Which episodes you want, whether to download them, how many at a time, DeadAir and pinning are yours, and change for nobody else. The feed’s address, folder, schedule and category, and marking it shareable again, are the same for everyone and an admin’s to change.</p>',
+          doc: '<p>Send only the fields to change. Which episodes you want, whether to download them, how many at a time, DeadAir, pinning and keeping a show from an OPML are yours, and change for nobody else. The feed’s address, folder, schedule and category, and marking it shareable again, are the same for everyone and an admin’s to change.</p>',
           pathParams: [['id', 'String', 'The feed’s id.']],
           body: 'FeedPatch', returns: '204 No Content',
           errors: [['400 Bad Request', 'A value out of range, such as a DeadAir pause that is not 200, 300, 400 or 500.'], ['403 Forbidden', 'A field only an admin may change.'], NOT_SUBSCRIBED],
@@ -477,6 +477,7 @@ export const ipx = {
       ['entries', 'i64', ''], ['downloaded', 'i64', ''], ['unread', 'i64', ''],
       ['subscribers', 'i64', 'People on this server, you included.'],
       ['pinned', 'bool', 'At the top of your list.'],
+      ['kept', 'bool', 'From an OPML, and yours to keep as a feed of its own should the OPML drop it or you leave the OPML.'],
       ['private', 'bool', 'Kept out of the directory.'],
       ['private_by_address', 'bool', 'Private whatever it is marked: a login, or a key in its address.'],
     ] },
@@ -493,6 +494,7 @@ export const ipx = {
       ['allow_explicit', 'Option<bool>', 'Yours.'], ['auto_download', 'Option<bool>', 'Yours.'],
       ['max_new_per_check', 'Option<Option<usize>>', 'Yours; <code>null</code> for the server’s.'],
       ['pinned', 'Option<bool>', 'Yours.'],
+      ['kept', 'Option<bool>', 'Yours: keep a show from an OPML should the OPML drop it.'],
       ['deadair', 'Option<Option<bool>>', 'Yours; <code>null</code> follows its OPML list’s.'],
       ['deadair_pause', 'Option<i64>', 'Yours: 200, 300, 400 or 500.'],
       ['private', 'Option<bool>', 'Anyone subscribed may mark it private; only an admin unmarks it.'],

tests/ui/app.spec.js

@@ -486,6 +486,43 @@ test('inside an OPML, feeds with unread items are listed first', async ({ page }
   await expect(page.locator('.childrow b')).toHaveText(want);
 });
 
+test('a show inside an OPML can be kept, and only it', async ({ page }) => {
+  await page.evaluate(() =>
+    fetch('/api/fetch', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ feed: 'test-subscriptions', force: true }),
+    }));
+  const chev = page.locator('.feed.group .chev');
+  await expect(chev).toBeVisible({ timeout: 20_000 });
+  if (await chev.getAttribute('aria-expanded') === 'false') await chev.click();
+  await page.locator('#feedlist').getByText('Aardvark Radio').click();
+  await page.locator('#content .acts [data-a="settings"]').click();
+
+  // Saved settings on a show would stop it following the OPML's for later tests, so the save is
+  // caught here and only what it would send is checked (#280).
+  let sent = null;
+  await page.route('**/api/feeds/*', (route) => {
+    if (route.request().method() !== 'PATCH') return route.continue();
+    sent = route.request().postDataJSON();
+    return route.fulfill({ status: 204 });
+  });
+  const keep = page.locator('#skept');
+  await expect(keep).not.toBeChecked();
+  await expect(page.locator('#modalCard')).toContainText('Test Subscriptions stops listing it');
+  await keep.check({ force: true });
+  await page.locator('#ssave').click();
+  await expect.poll(() => sent && sent.kept).toBe(true);
+  await page.unroute('**/api/feeds/*');
+
+  // The OPML's own settings have no Keep.
+  await page.keyboard.press('Escape');
+  await page.locator('.feed', { hasText: 'Test Subscriptions' }).first().click();
+  await page.locator('#content .acts [data-a="settings"]').click();
+  await expect(page.locator('#modalCard')).toBeVisible();
+  await expect(page.locator('#skept')).toHaveCount(0);
+});
+
 test('marking an OPML subscription read covers the feeds inside it', async ({ page }) => {
   await page.evaluate(() =>
     fetch('/api/fetch', {

web/CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- A show inside an OPML subscription has Keep in its settings, so it stays on your list as a show of its own if the OPML drops it or you unsubscribe from the OPML (#280).
 - Admin, Server: a setting to give everyone Signals on the server's TypeSafe key (#271).
 - Admin, Server: how many days back a new Signal reads (#276).
 

web/src/dialogs.ts

@@ -578,6 +578,9 @@ function settingsModal(f: Feed, newUrl?: string){
   const kids=S.feeds.filter(c=>c.group===f.id).length;
   const isGroup=kids>0;
   const parent=f.managed?S.feeds.find(p=>p.id===f.group):null;
+  // A show from an OPML or a Patreon creator, which you may keep should the list drop it (#280).
+  const list=f.managed&&!!f.group;
+  const listName=parent?(parent.title||parent.id):f.group||'';
   const admin=!!(S.me&&S.me.admin);
   const name=f.title||f.id;
   // Blank follows the server's default; 0 is every item (#166).
@@ -616,6 +619,9 @@ function settingsModal(f: Feed, newUrl?: string){
         <div class="dpslide"><input type="range" id="sdpause" min="200" max="500" step="100" value="${f.deadair_pause||300}"${f.deadair?'':' disabled'}></div>
         <div class="dpends" aria-hidden="true"><span>Brisk</span><span>Relaxed</span></div>
         <p class="cap">The silence kept where a long one is cut. More is easier on the ear; less saves more time.</p></div></div></div>
+    ${list?`<div class="ffield">${sw(`<input type="checkbox" role="switch" id="skept"${f.kept?' checked':''}>`,'Keep',' lead')}
+      <p class="cap">Stays on your list as a show of its own if ${esc(listName)} stops listing it, or if you
+      unsubscribe from ${esc(listName)}.</p></div>`:''}
     ${isGroup?'':`<div class="ffield"><label class="flabel" for="sblockin">Hide items that mention</label>
       <div class="chips" id="sblock">${chipsHTML(f.blocked||[],'sblockin')}</div>
       <p class="cap">Whole words or phrases. Hidden from your list and not downloaded for you, along
@@ -713,6 +719,8 @@ function settingsModal(f: Feed, newUrl?: string){
         auto_download:$<HTMLInputElement>('#sauto').checked, allow_explicit:$<HTMLInputElement>('#sexp').checked, deadair:$<HTMLInputElement>('#sdead').checked,
         deadair_pause:Number($<HTMLInputElement>('#sdpause').value)};
       if($('#sblock')) patch.blocked=chipWords($('#sblock'));
+      const kept=$<HTMLInputElement>('#skept');
+      if(kept&&kept.checked!==!!f.kept) patch.kept=kept.checked;
       // Sent only when it changed: anyone may mark a feed private, but only an admin unmark it.
       const priv=$<HTMLInputElement>('#sprivate');
       if(!priv.disabled&&priv.checked!==!!f.private) patch.private=priv.checked;

web/src/types.ts

@@ -35,6 +35,8 @@ interface Feed {
   unread: number;
   subscribers: number;
   pinned: boolean;
+  /** From an OPML, and yours to keep should it drop the feed (#280). */
+  kept: boolean;
   private: boolean;
   private_by_address: boolean;
 }