Step 5: quota and age retention
Oldest-first reaper with the original's 50 MB headroom pad, plus a reconcile pass for files deleted by hand and pruning of stale entries. The Python meant to reap only read, unflagged episodes but a missing import and a typo made that filter throw on every candidate. Requiring read=1 would be equally dead headless, so flagged is the keep-forever marker and read only decides ordering. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RPyeapneuXrCdojsaiXGbe
This commit is contained in:
187
src/retention.rs
Normal file
187
src/retention.rs
Normal file
@@ -0,0 +1,187 @@
|
||||
//! Disk quota and age sweeps. Replaces iPXQuotaManager.py ("SmartSpace").
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::db::{Candidate, Db, now};
|
||||
|
||||
/// Headroom kept below the quota, as getToQuota()'s `1048576 * 50` did.
|
||||
pub const PAD_BYTES: u64 = 50 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Report {
|
||||
pub reconciled: usize,
|
||||
pub aged_out: Vec<Candidate>,
|
||||
pub over_quota: Vec<Candidate>,
|
||||
pub entries_pruned: usize,
|
||||
pub bytes_freed: u64,
|
||||
}
|
||||
|
||||
/// Oldest-first until enough is freed. Pure, so the ordering rule is testable without a disk.
|
||||
pub fn pick(candidates: &[Candidate], total: u64, limit: u64) -> Vec<Candidate> {
|
||||
let ceiling = limit.saturating_sub(PAD_BYTES);
|
||||
if total <= ceiling {
|
||||
return vec![];
|
||||
}
|
||||
let mut need = total - ceiling;
|
||||
let mut out = vec![];
|
||||
for c in candidates {
|
||||
if need == 0 {
|
||||
break;
|
||||
}
|
||||
let size = c.bytes.max(0) as u64;
|
||||
out.push(c.clone());
|
||||
need = need.saturating_sub(size);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Files older than the cutoff. Flagged ones are already excluded by the query.
|
||||
pub fn aged(candidates: &[Candidate], cutoff: i64) -> Vec<Candidate> {
|
||||
candidates
|
||||
.iter()
|
||||
.filter(|c| c.age_key > 0 && c.age_key < cutoff)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
|
||||
let mut report = Report::default();
|
||||
|
||||
// Someone may have deleted a file by hand; the row must stop claiming it exists.
|
||||
for (id, path) in db.missing_files()? {
|
||||
if !dry_run {
|
||||
db.mark_reaped(id)?;
|
||||
}
|
||||
tracing::debug!(path, "file gone, row reaped");
|
||||
report.reconciled += 1;
|
||||
}
|
||||
|
||||
let candidates = db.reap_candidates()?;
|
||||
|
||||
if cfg.general.max_age_days > 0 {
|
||||
let cutoff = now() - (cfg.general.max_age_days * 86_400) as i64;
|
||||
report.aged_out = aged(&candidates, cutoff);
|
||||
for c in &report.aged_out {
|
||||
report.bytes_freed += remove(db, c, dry_run)?;
|
||||
}
|
||||
if !dry_run {
|
||||
report.entries_pruned = db.prune_entries(cutoff)?;
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.general.max_total_gb > 0.0 {
|
||||
let limit = (cfg.general.max_total_gb * 1_073_741_824.0) as u64;
|
||||
// Re-read: the age sweep may already have freed enough.
|
||||
let remaining: Vec<Candidate> = candidates
|
||||
.iter()
|
||||
.filter(|c| !report.aged_out.iter().any(|a| a.id == c.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
let total: u64 = remaining.iter().map(|c| c.bytes.max(0) as u64).sum();
|
||||
report.over_quota = pick(&remaining, total, limit);
|
||||
for c in &report.over_quota {
|
||||
report.bytes_freed += remove(db, c, dry_run)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
|
||||
if dry_run {
|
||||
return Ok(c.bytes.max(0) as u64);
|
||||
}
|
||||
let size = std::fs::metadata(&c.path).map(|m| m.len()).unwrap_or(c.bytes.max(0) as u64);
|
||||
// Missing is fine: the point is that it is gone.
|
||||
if let Err(e) = std::fs::remove_file(&c.path)
|
||||
&& e.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!(path = c.path, error = %e, "could not delete");
|
||||
return Ok(0);
|
||||
}
|
||||
db.mark_reaped(c.id)?;
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cand(id: i64, bytes: i64, age_key: i64, read: bool) -> Candidate {
|
||||
Candidate {
|
||||
id,
|
||||
url: format!("https://x/{id}"),
|
||||
path: format!("/tmp/{id}"),
|
||||
bytes,
|
||||
age_key,
|
||||
read,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_takes_oldest_first_and_stops_once_under() {
|
||||
let gb = 1_073_741_824u64;
|
||||
// Ten 1 GB files, oldest first as the query would return them.
|
||||
let cands: Vec<Candidate> = (1..=10).map(|i| cand(i, gb as i64, i, true)).collect();
|
||||
|
||||
// 10 GB used against a 6 GB quota: need 4 GB back, plus the 50 MB pad.
|
||||
let picked = pick(&cands, 10 * gb, 6 * gb);
|
||||
assert_eq!(
|
||||
picked.iter().map(|c| c.id).collect::<Vec<_>>(),
|
||||
vec![1, 2, 3, 4, 5],
|
||||
"oldest first, one extra to clear the headroom pad"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pick_deletes_nothing_when_under_quota() {
|
||||
let gb = 1_073_741_824u64;
|
||||
let cands: Vec<Candidate> = (1..=3).map(|i| cand(i, gb as i64, i, true)).collect();
|
||||
assert!(pick(&cands, 3 * gb, 50 * gb).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aged_only_takes_files_past_the_cutoff() {
|
||||
let cands = vec![cand(1, 10, 100, true), cand(2, 10, 500, true), cand(3, 10, 0, true)];
|
||||
let picked = aged(&cands, 200);
|
||||
assert_eq!(picked.iter().map(|c| c.id).collect::<Vec<_>>(), vec![1]);
|
||||
// age_key 0 means "never recorded" -- not the same as "infinitely old".
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_never_offers_flagged_files_and_prefers_read_ones() {
|
||||
let db = Db::memory().unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO entries (feed_id, guid, first_seen, read, flagged) VALUES
|
||||
('f', 'keep', 0, 1, 1),
|
||||
('f', 'unread', 0, 0, 0),
|
||||
('f', 'read', 0, 1, 0);
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, path, bytes_done, state, downloaded_at) VALUES
|
||||
(1, 'f', 'keep', 'u1', '/tmp/keep', 10, 'done', 10),
|
||||
(2, 'f', 'unread', 'u2', '/tmp/unread', 10, 'done', 20),
|
||||
(3, 'f', 'read', 'u3', '/tmp/read', 10, 'done', 30);",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let got: Vec<i64> = db.reap_candidates().unwrap().iter().map(|c| c.id).collect();
|
||||
assert_eq!(got, vec![3, 2], "flagged excluded; read goes before unread");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_keeps_entries_that_still_have_a_file() {
|
||||
let db = Db::memory().unwrap();
|
||||
db.exec_for_test(
|
||||
"INSERT INTO entries (feed_id, guid, first_seen, read, flagged) VALUES
|
||||
('f', 'has-file', 100, 1, 0),
|
||||
('f', 'no-file', 100, 1, 0),
|
||||
('f', 'flagged', 100, 1, 1),
|
||||
('f', 'recent', 900, 1, 0);
|
||||
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
|
||||
(1, 'f', 'has-file', 'u1', '/tmp/x', 'done');",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.prune_entries(500).unwrap(), 1, "only the old, fileless, unflagged one");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user