Insert only a feed's new items and files on a scan (#96)
The spans added in 2799704 showed it: in a full scan of 134 feeds (trace da9a419b...,
2026-09-29 17:31, 315 s), storing items took 117 s, fetching 44 s and every other database call
about 2 s together. A scan inserted every item and file the feed listed, stored or not, one
round trip of about 10 ms each; Clarkesworld's 1200 items took 13 s. It now reads the feed's
stored guids and file URLs once (Db::stored_items) and inserts only the rest. A file URL not
among the feed's own may still be another feed's, so that one still goes to the insert, which
finds it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1316,16 +1316,19 @@ async fn scan_one(
|
||||
// discovery, it outlived the setting behind it, and allowing explicit items afterwards
|
||||
// changed nothing however often the feed was scanned.
|
||||
let skipped = ctx.db.skipped_by_filter(id).await?;
|
||||
let (known_items, known_files) = ctx.db.stored_items(id).await?;
|
||||
let mut scan = Scan::default();
|
||||
// Its own span: the time a feed spends after its fetch was untraced (#96).
|
||||
let store = tracing::info_span!("store", items = parsed.entries.len());
|
||||
tracing::Instrument::instrument(async {
|
||||
for entry in &parsed.entries {
|
||||
if ctx.db.record_entry(id, entry).await? {
|
||||
// Only what is not stored yet is inserted; the insert would find the rest and do nothing.
|
||||
if !known_items.contains(&entry.guid) && ctx.db.record_entry(id, entry).await? {
|
||||
scan.new_entries += 1;
|
||||
}
|
||||
for enc in &entry.enclosures {
|
||||
let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
|
||||
// A URL not among this feed's files may still be another feed's: the insert says.
|
||||
let was = if !known_files.contains(&enc.url) && ctx.db.record_enclosure(id, &entry.guid, enc).await? {
|
||||
None
|
||||
} else if let Some(reason) = skipped.get(&enc.url) {
|
||||
Some(reason.as_str())
|
||||
|
||||
Reference in New Issue
Block a user