List feeds in the Directory before anyone subscribes, and clear out dead ones (#107)
The Directory showed a catalogue feed only once someone subscribed, and a feed left the catalogue with its last subscriber, so nothing could be put there for others to find. - A feed has a listed flag, set by ipx add --list (with --category for the Directory's chip). The web page keeps a listed feed in the catalogue when its last subscriber leaves. - The Directory lists every catalogue feed; Popular still only what people subscribe to. popular() reads titles, artwork and categories through Db::feed_list, not three queries a feed. Subscribing from the Directory scans the feed at once. - A feed nobody subscribes to is checked once a day at most. - clean_directory, in the sweep before each scan, removes from the catalogue and the database a feed nobody subscribes to, with no file on disk and not from an OPML, that has failed for 30 days or published nothing in a year. Run against production first: it removes nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
66
src/db.rs
66
src/db.rs
@@ -778,13 +778,47 @@ impl Db {
|
||||
.map(|r| Ok(r.try_get("", "id")?))
|
||||
.collect::<Result<_>>()?;
|
||||
for id in &gone {
|
||||
for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
|
||||
self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.clone().into()]).await?;
|
||||
}
|
||||
self.forget_rows(id).await?;
|
||||
}
|
||||
Ok(gone)
|
||||
}
|
||||
|
||||
/// Everything stored about a feed but its row: items, file rows, read state, block list.
|
||||
async fn forget_rows(&self, id: &str) -> Result<()> {
|
||||
for table in ["enclosures", "entry_state", "hidden", "blocklists", "entries"] {
|
||||
self.exec(&format!("DELETE FROM {table} WHERE feed_id = $1"), vec![id.into()]).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forgets a feed and everything stored about it.
|
||||
pub async fn forget_feed(&self, id: &str) -> Result<()> {
|
||||
self.forget_rows(id).await?;
|
||||
self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Feeds nobody subscribes to that are dead, failing since before `dead_before`, or quiet,
|
||||
/// their newest item from before `quiet_before`: what keeps the Directory to feeds worth
|
||||
/// taking. One with a file on disk stays, and so does one an OPML subscription lists, which
|
||||
/// comes back with the OPML anyway. A feed with no items yet is not quiet, only new.
|
||||
pub async fn stale_unsubscribed(&self, dead_before: i64, quiet_before: i64) -> Result<Vec<String>> {
|
||||
self.rows(
|
||||
"SELECT f.id FROM feeds f
|
||||
WHERE NOT EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = f.id AND x.path IS NOT NULL)
|
||||
AND f.group_id IS NULL AND NOT coalesce(f.managed, false)
|
||||
AND ((f.error_since IS NOT NULL AND f.error_since < $1)
|
||||
OR (SELECT max(coalesce(e.published, e.first_seen)) FROM entries e
|
||||
WHERE e.feed_id = f.id) < $2)",
|
||||
vec![dead_before.into(), quiet_before.into()],
|
||||
)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| Ok(r.try_get("", "id")?))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn prune_entries(&self, older_than: i64) -> Result<usize> {
|
||||
let n = self
|
||||
.exec(
|
||||
@@ -1998,6 +2032,32 @@ mod tests {
|
||||
assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_feeds_nobody_subscribes_to_are_found_and_the_rest_left() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true);
|
||||
INSERT INTO feeds (id, url, error_since, managed, group_id) VALUES
|
||||
('dead','a',100,false,null),('quiet','b',null,false,null),('news','c',null,false,null),
|
||||
('new','d',null,false,null),('wanted','e',100,false,null),('kept','f',100,false,null),
|
||||
('child','g',100,true,'opml'),('lately','h',900,false,null);
|
||||
INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'wanted');
|
||||
INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
|
||||
('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100);
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');",
|
||||
).await
|
||||
.unwrap();
|
||||
// Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100.
|
||||
// news published lately, new has no items yet, wanted has a subscriber, kept a file,
|
||||
// child comes from an OPML, lately began failing after the cutoff.
|
||||
let mut stale = db.stale_unsubscribed(500, 500).await.unwrap();
|
||||
stale.sort();
|
||||
assert_eq!(stale, ["dead", "quiet"]);
|
||||
db.forget_feed("dead").await.unwrap();
|
||||
assert_eq!(db.feed_summary("dead").await.unwrap().entries, 0);
|
||||
assert!(!db.feed_urls().await.unwrap().iter().any(|(id, _)| id == "dead"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_feed_nobody_subscribes_to_is_forgotten() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user