Insert only a feed's new items and files on a scan (#96)

The spans added in 2799704 showed it: in a full scan of 134 feeds (trace da9a419b...,
2026-09-29 17:31, 315 s), storing items took 117 s, fetching 44 s and every other database call
about 2 s together. A scan inserted every item and file the feed listed, stored or not, one
round trip of about 10 ms each; Clarkesworld's 1200 items took 13 s. It now reads the feed's
stored guids and file URLs once (Db::stored_items) and inserts only the rest. A file URL not
among the feed's own may still be another feed's, so that one still goes to the insert, which
finds it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Checking a feed with a long history is much quicker: items already stored are no longer written again on every check.
 - A scan no longer asks a feed's website for its icon every time; it asks again when the artwork changes or you refresh the feed.
 - The feed list loads several times faster: it was asking the database six questions per feed.
 - Artwork a feed offers only over plain http shows on the https site too.

src/db.rs

@@ -1779,6 +1779,27 @@ impl Db {
     /// A feed's enclosures skipped by one of its filters, by URL, with the reason: the verdicts a
     /// change of settings can overturn. A torrent held back while torrents are off is not a
     /// filter's call.
+    /// The guids of a feed's stored items and the URLs of its files: a scan inserts only what is
+    /// not among them. Inserting every item it read, stored or not, cost a round trip each, about
+    /// 10 ms: 13 s a scan for Clarkesworld's 1200 items, and 117 s of a full 315 s scan (#96).
+    #[tracing::instrument(skip_all)]
+    pub async fn stored_items(
+        &self,
+        feed_id: &str,
+    ) -> Result<(std::collections::HashSet<String>, std::collections::HashSet<String>)> {
+        let col = |sql: &'static str, name: &'static str| async move {
+            self.rows(sql, vec![feed_id.into()])
+                .await?
+                .iter()
+                .map(|r| Ok(r.try_get::<String>("", name)?))
+                .collect::<Result<std::collections::HashSet<_>>>()
+        };
+        Ok((
+            col("SELECT guid FROM entries WHERE feed_id = $1", "guid").await?,
+            col("SELECT url FROM enclosures WHERE feed_id = $1", "url").await?,
+        ))
+    }
+
     #[tracing::instrument(skip_all)]
     pub async fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> {
         self.rows(
@@ -1913,6 +1934,19 @@ mod tests {
         assert_eq!((list["f"].unread, list["g"].unread), (1, 1)); // a read, c hidden; sam's read is sam's
     }
 
+    #[tokio::test]
+    async fn a_scan_knows_a_feeds_stored_items_and_files() {
+        let db = Db::memory().await.unwrap();
+        db.exec_for_test(
+            "INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0),('f','b',0),('g','c',0);
+             INSERT INTO enclosures (id, feed_id, guid, url, state) VALUES (1,'f','a','u1','pending'),(2,'g','c','u2','pending');",
+        ).await
+        .unwrap();
+        let (items, files) = db.stored_items("f").await.unwrap();
+        assert_eq!(items, ["a", "b"].map(String::from).into());
+        assert_eq!(files, ["u1"].map(String::from).into()); // u2 is g's: left to the insert to find
+    }
+
     #[tokio::test]
     async fn only_artwork_a_feed_names_is_fetched_for_the_page() {
         let db = Db::memory().await.unwrap();

src/main.rs

@@ -1316,16 +1316,19 @@ async fn scan_one(
     // discovery, it outlived the setting behind it, and allowing explicit items afterwards
     // changed nothing however often the feed was scanned.
     let skipped = ctx.db.skipped_by_filter(id).await?;
+    let (known_items, known_files) = ctx.db.stored_items(id).await?;
     let mut scan = Scan::default();
     // Its own span: the time a feed spends after its fetch was untraced (#96).
     let store = tracing::info_span!("store", items = parsed.entries.len());
     tracing::Instrument::instrument(async {
         for entry in &parsed.entries {
-            if ctx.db.record_entry(id, entry).await? {
+            // Only what is not stored yet is inserted; the insert would find the rest and do nothing.
+            if !known_items.contains(&entry.guid) && ctx.db.record_entry(id, entry).await? {
                 scan.new_entries += 1;
             }
             for enc in &entry.enclosures {
-                let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? {
+                // A URL not among this feed's files may still be another feed's: the insert says.
+                let was = if !known_files.contains(&enc.url) && ctx.db.record_enclosure(id, &entry.guid, enc).await? {
                     None
                 } else if let Some(reason) = skipped.get(&enc.url) {
                     Some(reason.as_str())