Clear dead feeds from the Directory once a day, not before every scan (#193)
stale_unsubscribed takes the newest item of every unsubscribed feed, 2-5 s warm and up to 12 s in production, and reap ran it before each scan, every few minutes, holding up the sequential worker's scans and downloads. Its thresholds are a month dead and a year quiet, so a day is plenty. Kept in memory: a restart looks again at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
28
src/main.rs
28
src/main.rs
@@ -156,6 +156,9 @@ pub struct Ctx {
|
||||
/// Wakes DeadAir's maker when a show is turned on, so its copies start at once instead of
|
||||
/// at the next pass (#171).
|
||||
pub deadair: tokio::sync::Notify,
|
||||
/// When the Directory was last cleared of dead feeds (`clean_directory`), 0 for not since
|
||||
/// starting.
|
||||
pub directory_cleaned: std::sync::atomic::AtomicI64,
|
||||
}
|
||||
|
||||
/// A copy of the config to change, holding off every other change until `store_cfg` keeps it.
|
||||
@@ -348,6 +351,7 @@ async fn main() -> Result<()> {
|
||||
detach_torrents: is_daemon,
|
||||
retry: Default::default(),
|
||||
cfg_write: Default::default(),
|
||||
directory_cleaned: Default::default(),
|
||||
deadair: Default::default(),
|
||||
});
|
||||
|
||||
@@ -1118,6 +1122,13 @@ async fn clean_directory(ctx: &Ctx) -> Result<()> {
|
||||
const DEAD: i64 = 30 * 86_400;
|
||||
const QUIET: i64 = 365 * 86_400;
|
||||
let now = db::now();
|
||||
// Once a day, for feeds dead a month or quiet a year: before every scan, its query took 1 to
|
||||
// 12 s every few minutes, and the scans and downloads waited behind it (#193).
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
if now - ctx.directory_cleaned.load(Relaxed) < 86_400 {
|
||||
return Ok(());
|
||||
}
|
||||
ctx.directory_cleaned.store(now, Relaxed);
|
||||
let stale = ctx.db.stale_unsubscribed(now - DEAD, now - QUIET).await?;
|
||||
if stale.is_empty() {
|
||||
return Ok(());
|
||||
@@ -2467,9 +2478,26 @@ mod tests {
|
||||
retry: Default::default(),
|
||||
cfg_write: Default::default(),
|
||||
deadair: Default::default(),
|
||||
directory_cleaned: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dead_feeds_leave_the_directory_once_a_day_not_before_every_scan() {
|
||||
let ctx = test_ctx(config::Config::default()).await;
|
||||
let dead = |id: &str| format!("INSERT INTO feeds (id, url, error_since, managed) VALUES ('{id}', 'u-{id}', 100, false)");
|
||||
let listed = async || ctx.db.feed_urls().await.unwrap().into_iter().map(|(id, _)| id).collect::<Vec<_>>();
|
||||
ctx.db.sql(&dead("a")).await.unwrap();
|
||||
clean_directory(&ctx).await.unwrap();
|
||||
assert!(listed().await.is_empty(), "the first pass since starting looks");
|
||||
ctx.db.sql(&dead("b")).await.unwrap();
|
||||
clean_directory(&ctx).await.unwrap();
|
||||
assert_eq!(listed().await, ["b"], "not again the same day (#193)");
|
||||
ctx.directory_cleaned.store(db::now() - 86_401, std::sync::atomic::Ordering::Relaxed);
|
||||
clean_directory(&ctx).await.unwrap();
|
||||
assert!(listed().await.is_empty(), "a day on, it looks again");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_removed_account_takes_the_feeds_only_it_read_with_it() {
|
||||
let ctx = test_ctx(config::Config::default()).await;
|
||||
|
||||
Reference in New Issue
Block a user