diff --git a/PROGRESS.md b/PROGRESS.md index 0273a5b..7068456 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -56,6 +56,37 @@ and until now nothing set them. --- +## 2026-09-10 — Database cleanup, and the 304 trap it walked into + +Cleaned up on request: removed the `CT Log Archive Torrents` folder (123 preallocated files from the +abandoned 1.6 TB tuscolo torrent, **19 GB real** on disk, no row referencing any of it, feed no +longer subscribed), the unsubscribed ct-log feed's rows, and every OPML-derived entry/enclosure that +held no file so a rescan could rebuild them with the current parser. Kept every row holding a file, +so nothing on disk was orphaned, and kept the skipped/reaped history, without which the next scan +would re-download the 149 images just deleted. Disk 22 GB -> 3.8 GB, database 22.8 MB -> 5.2 MB. + +Correction worth recording: mid-download I checked that torrent folder, saw `du -sh` report 8 KB, +and told Ray it was sparse with nothing written. By the time it was abandoned it had allocated 19 GB. +The reassurance had a shelf life I did not mention. + +**Then "Abort Retry Fail is empty".** The cleanup deleted entries but left each feed's +ETag/Last-Modified. The rescan sent them, servers answered 304 (120 times in the log), the daemon +skipped parsing, and 57 feeds stayed empty -- and would have until a publisher happened to change +something. Cleared the validators on the empty feeds and rescanned: entries 1524 -> 3009, feeds with +content 19 -> 64, Abort Retry Fail back to 20. + +Fixed in code so it cannot recur: a 304 arriving while the feed has **zero stored entries** means the +validator has outlived the data -- a restore, a manual edit, a cleanup. The daemon now believes the +database over the validator, drops it and asks again. Proved live by deleting a feed's entries, +leaving its ETag, and rescanning: 20 entries rebuilt, self-heal logged once. + +**"not a wanted media type"** was internal jargon reaching the UI, and it was stored in `last_error` +so an ordinary filter decision rendered in red as though something had failed. Reworded to "not +audio or video", 144 existing rows updated, and the UI now paints a reason red only when the state +is actually `error`. + +--- + ## 2026-09-10 — Multiple enclosures per item, and viewing without downloading **View without downloading.** A non-audio/video enclosure now carries a View link opening in a new diff --git a/src/main.rs b/src/main.rs index 84d545a..b977e66 100644 --- a/src/main.rs +++ b/src/main.rs @@ -795,7 +795,7 @@ async fn scan_one( feed_cfg: &config::Feed, state: &db::HttpState, ) -> Result { - let fetched = feed::fetch( + let mut fetched = feed::fetch( &ctx.client, feed_cfg, state.etag.as_deref(), @@ -803,6 +803,16 @@ async fn scan_one( ) .await?; + // A 304 while nothing is stored means the validator has outlived the data -- a restore + // from backup, a manual edit, a cleanup that removed entries. Believe the database over + // the validator: drop it and ask again, or the feed stays empty until the publisher + // happens to change something. + if matches!(fetched, feed::Fetched::NotModified) && ctx.db.feed_summary(id)?.entries == 0 { + tracing::info!(feed = id, "not modified, but nothing stored; refetching without the validator"); + ctx.db.clear_validators(id)?; + fetched = feed::fetch(&ctx.client, feed_cfg, None, None).await?; + } + let (bytes, etag, last_modified) = match fetched { feed::Fetched::NotModified => { ctx.db.touch_feed(id, &feed_cfg.url)?; @@ -1011,7 +1021,7 @@ fn reject( .as_deref() .unwrap_or(&cfg.general.media_types); if !config::wanted_media(enc.mime.as_deref(), wanted) { - return Some("not a wanted media type"); + return Some("not audio or video"); } if entry.explicit && !feed_cfg.allow_explicit { return Some("explicit"); diff --git a/web/index.html b/web/index.html index d2e5884..308f35a 100644 --- a/web/index.html +++ b/web/index.html @@ -646,7 +646,8 @@ function epEl(e){ ${e.flagged?'★ kept':''} ${enc&&!has?`
`:''} - ${enc&&enc.last_error?`
${esc(enc.last_error)}
`:''} + ${enc&&enc.last_error&&enc.state==='error' + ?`
${esc(enc.last_error)}
`:''}
${playable?``: @@ -780,7 +781,8 @@ function encBox(x){ return `
${x.state==='skipped'?kindOf(x):esc(x.state)} ${esc(kindOf(x))}${size?' \u00b7 '+size:''} - ${x.last_error?`${esc(x.last_error)}`:''} + ${x.last_error&&x.state==='error'?`${esc(x.last_error)}` + :(x.last_error?`${esc(x.last_error)}`:'')} ${viewable?`View`:''}
`;