List feeds in the Directory before anyone subscribes, and clear out dead ones (#107)

The Directory showed a catalogue feed only once someone subscribed, and a feed left the
catalogue with its last subscriber, so nothing could be put there for others to find.

- A feed has a listed flag, set by ipx add --list (with --category for the Directory's chip).
  The web page keeps a listed feed in the catalogue when its last subscriber leaves.
- The Directory lists every catalogue feed; Popular still only what people subscribe to.
  popular() reads titles, artwork and categories through Db::feed_list, not three queries a
  feed. Subscribing from the Directory scans the feed at once.
- A feed nobody subscribes to is checked once a day at most.
- clean_directory, in the sweep before each scan, removes from the catalogue and the database
  a feed nobody subscribes to, with no file on disk and not from an OPML, that has failed for
  30 days or published nothing in a year. Run against production first: it removes nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 21:13:13 +00:00
parent 65fdab8b13
commit 43acc62259
7 changed files with 161 additions and 17 deletions

View File

@@ -778,13 +778,47 @@ impl Db {
.map(|r| Ok(r.try_get("", "id")?))
.collect::<Result<_>>()?;
for id in &gone {
for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.clone().into()]).await?;
}
self.forget_rows(id).await?;
}
Ok(gone)
}
/// Everything stored about a feed but its row: items, file rows, read state, block list.
async fn forget_rows(&self, id: &str) -> Result<()> {
for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.into()]).await?;
}
Ok(())
}
/// Forgets a feed and everything stored about it.
pub async fn forget_feed(&self, id: &str) -> Result<()> {
self.forget_rows(id).await?;
self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?;
Ok(())
}
/// Feeds nobody subscribes to that are dead, failing since before `dead_before`, or quiet,
/// their newest item from before `quiet_before`: what keeps the Directory to feeds worth
/// taking. One with a file on disk stays, and so does one an OPML subscription lists, which
/// comes back with the OPML anyway. A feed with no items yet is not quiet, only new.
pub async fn stale_unsubscribed(&self, dead_before: i64, quiet_before: i64) -> Result<Vec<String>> {
self.rows(
"SELECT f.id FROM feeds f
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id)
AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = f.id AND x.path IS NOT NULL)
AND f.group_id IS NULL AND NOT coalesce(f.managed, false)
AND ((f.error_since IS NOT NULL AND f.error_since < $1)
OR (SELECT max(coalesce(e.published, e.first_seen)) FROM entries e
WHERE e.feed_id = f.id) < $2)",
vec![dead_before.into(), quiet_before.into()],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "id")?))
.collect()
}
pub async fn prune_entries(&self, older_than: i64) -> Result<usize> {
let n = self
.exec(
@@ -1998,6 +2032,32 @@ mod tests {
assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
}
#[tokio::test]
async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() {
let db = Db::memory().await.unwrap();
db.exec_for_test(
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
INSERT INTO feeds (id, url, error_since, managed, group_id) VALUES
('dead','a',100,false,null),('quiet','b',null,false,null),('news','c',null,false,null),
('new','d',null,false,null),('wanted','e',100,false,null),('kept','f',100,false,null),
('child','g',100,true,'opml'),('lately','h',900,false,null);
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted');
INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');",
).await
.unwrap();
// Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100.
// news published lately, new has no items yet, wanted has a subscriber, kept a file,
// child comes from an OPML, lately began failing after the cutoff.
let mut stale = db.stale_unsubscribed(500, 500).await.unwrap();
stale.sort();
assert_eq!(stale, ["dead", "quiet"]);
db.forget_feed("dead").await.unwrap();
assert_eq!(db.feed_summary("dead").await.unwrap().entries, 0);
assert!(!db.feed_urls().await.unwrap().iter().any(|(id, _)| id == "dead"));
}
#[tokio::test]
async fn a_failing_feed_nobody_subscribes_to_is_forgotten() {
let db = Db::memory().await.unwrap();