Insert only a feed's new items and files on a scan (#96)

The spans added in 2799704 showed it: in a full scan of 134 feeds (trace da9a419b...,
2026-09-29 17:31, 315 s), storing items took 117 s, fetching 44 s and every other database call
about 2 s together. A scan inserted every item and file the feed listed, stored or not, one
round trip of about 10 ms each; Clarkesworld's 1200 items took 13 s. It now reads the feed's
stored guids and file URLs once (Db::stored_items) and inserts only the rest. A file URL not
among the feed's own may still be another feed's, so that one still goes to the insert, which
finds it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 17:40:52 +00:00
parent 2799704d30
commit 57ab419718
3 changed files with 40 additions and 2 deletions

View File

@@ -1316,16 +1316,19 @@ async fn scan_one(
// discovery, it outlived the setting behind it, and allowing explicit items afterwards
// changed nothing however often the feed was scanned.
let skipped = ctx.db.skipped_by_filter(id).await?;
let (known_items, known_files) = ctx.db.stored_items(id).await?;
let mut scan = Scan::default();
// Its own span: the time a feed spends after its fetch was untraced (#96).
let store = tracing::info_span!("store", items = parsed.entries.len());
tracing::Instrument::instrument(async {
for entry in &parsed.entries {
if ctx.db.record_entry(id, entry).await? {
// Only what is not stored yet is inserted; the insert would find the rest and do nothing.
if !known_items.contains(&entry.guid) && ctx.db.record_entry(id, entry).await? {
scan.new_entries += 1;
}
for enc in &entry.enclosures {
let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
// A URL not among this feed's files may still be another feed's: the insert says.
let was = if !known_files.contains(&enc.url) && ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())