Compressed answers, a directory twice as patient, fewer whole re-reads (#239, #242, #243, #244)

Measured on 2026-10-15: the feed list and item lists went out uncompressed at over 100 KB; ipx
read 2.6 GB of feed bodies a day and the directory 3.9 GB.

- ipx-web compresses with zstd or gzip, leaving out /api/events, which a compressor would hold
  back, and /media (#239).
- A body identical to the last one (SHA-256 in feeds.body_hash) is taken as a 304, unless
  something is missing a read would fill or the check is forced; and a feed without a schedule
  of its own whose body has been the same waits a twelfth of that, up to four hours, beside the
  failure backoff (#242). 35 Patreon show feeds sent ~300 KB each hourly, about 250 MB a day.
- The directory checks each feed every twelve hours, not six (#243).
- reqwest accepts brotli and zstd too (#244).

#240 (cache the feed list's counts) is left open: at production's size on the test Postgres
that count takes 12 ms and All Subscriptions 32 ms, against 366 and 300 ms in production, so
the cause is production's database, not the query (#241).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/directory/CHANGELOG.md

@@ -10,6 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Each feed is checked every twelve hours instead of six (#243).
+- Feeds are fetched accepting brotli and zstd as well as gzip (#244).
+
 ## [1.1.1] - 2026-10-15
 
 ### Changed

crates/directory/src/lib.rs

@@ -117,8 +117,8 @@ pub struct Blocked {
 pub const MAX_SNAPSHOT: usize = 20_000;
 
 /// How often each feed is fetched again. Metadata changes slowly, and every server subscribed
-/// to a feed fetches it itself anyway.
-pub const REFRESH_EVERY: i64 = 6 * 3600;
+/// to a feed fetches it itself anyway: every six hours was 3.9 GB of feeds a day (#243).
+pub const REFRESH_EVERY: i64 = 12 * 3600;
 
 pub struct Directory {
     pub store: Store,

crates/ipx/CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- A feed that sends the same body again, as Patreon and Substack do whatever they are asked, is taken as unchanged instead of read again, and, without a schedule of its own, checked less often the longer it stays the same, up to every four hours, back on schedule when it changes (#242).
+- Feeds are fetched accepting brotli and zstd as well as gzip (#244).
+
 ## [1.2.0] - 2026-10-15
 
 ### Added

crates/ipx/src/db.rs

@@ -95,6 +95,11 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
             orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
         }
     }
+    if orm.execute_unprepared("SELECT body_hash FROM feeds LIMIT 0").await.is_err() {
+        for sql in ["ALTER TABLE feeds ADD COLUMN body_hash TEXT", "ALTER TABLE feeds ADD COLUMN same_since BIGINT"] {
+            orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
+        }
+    }
     if orm.execute_unprepared("SELECT pause_ms FROM deadair_prefs LIMIT 0").await.is_err() {
         let sql = "ALTER TABLE deadair_prefs ADD COLUMN pause_ms BIGINT";
         orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
@@ -294,6 +299,8 @@ pub struct FeedSummary {
     pub last_error: Option<String>,
     /// When this run of failures began; see the `error_since` column.
     pub error_since: Option<i64>,
+    /// Since when the feed has sent the same body (#242).
+    pub same_since: Option<i64>,
     pub entries: i64,
     pub downloaded: i64,
 }
@@ -499,6 +506,7 @@ impl Db {
                         last_error: f.last_error,
                         orphaned: f.orphaned,
                         error_since: f.error_since,
+                        same_since: f.same_since,
                         category: f.category,
                     },
                     ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64),
@@ -530,6 +538,9 @@ pub struct HttpState {
     pub ttl_mins: Option<u64>,
     /// When the feed's current run of failures began, for backing off (`due_after`).
     pub error_since: Option<i64>,
+    /// The last body's hash, and since when it has been that body (#242).
+    pub body_hash: Option<String>,
+    pub same_since: Option<i64>,
 }
 
 impl From<feeds::Model> for HttpState {
@@ -540,6 +551,8 @@ impl From<feeds::Model> for HttpState {
             last_checked: f.last_checked,
             ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64),
             error_since: f.error_since,
+            body_hash: f.body_hash,
+            same_since: f.same_since,
         }
     }
 }
@@ -606,6 +619,16 @@ impl Db {
     }
 
     /// 304, or any other poll that produced no new data: only the clock moves.
+    /// A body read whole that was not the last one: its hash, and that it changed now (#242).
+    pub async fn set_body(&self, feed_id: &str, hash: &str) -> Result<()> {
+        self.exec(
+            "UPDATE feeds SET body_hash = $2, same_since = $3 WHERE id = $1",
+            vec![feed_id.into(), hash.into(), now().into()],
+        )
+        .await?;
+        Ok(())
+    }
+
     pub async fn touch_feed(&self, feed_id: &str, url: &str) -> Result<()> {
         self.exec(
             "INSERT INTO feeds (id, url, last_checked) VALUES ($1, $2, $3)
@@ -2581,6 +2604,23 @@ mod tests {
         assert_eq!(db.http_state("f").await.unwrap().etag.as_deref(), Some("\"b\""));
     }
 
+    #[tokio::test]
+    async fn a_database_from_before_body_hashes_gains_the_columns() {
+        let db = Db::memory().await.unwrap();
+        db.record_feed("f", "https://x/f", Some("F"), None, None, None, None, None, None).await.unwrap();
+        db.set_body("f", "abc").await.unwrap();
+        assert_eq!(db.http_state("f").await.unwrap().body_hash.as_deref(), Some("abc"));
+        // A database from before #242.
+        for sql in ["ALTER TABLE feeds DROP COLUMN body_hash", "ALTER TABLE feeds DROP COLUMN same_since"] {
+            db.exec_for_test(sql).await.unwrap();
+        }
+        create_missing(&db.orm).await.unwrap();
+        let st = db.http_state("f").await.unwrap();
+        assert_eq!((st.body_hash, st.same_since), (None, None), "every feed is read whole once more");
+        db.set_body("f", "def").await.unwrap();
+        assert_eq!(db.http_state("f").await.unwrap().body_hash.as_deref(), Some("def"));
+    }
+
     #[tokio::test]
     async fn every_sort_column_runs_and_orders_both_ways() {
         let db = Db::memory().await.unwrap();

crates/ipx/src/entity.rs

@@ -46,6 +46,12 @@ pub mod feeds {
         /// are re-derived instead. Customising one promotes it to config.
         #[sea_orm(default_value = false)]
         pub managed: bool,
+        /// SHA-256 of the body last read, in hex: the same body again is read as "not modified"
+        /// (#242). Added after the table was, so `create_missing` adds it, and `same_since`.
+        #[sea_orm(column_type = "Text", nullable)]
+        pub body_hash: Option<String>,
+        /// When the body last changed: the longer ago, the less often the feed is checked (#242).
+        pub same_since: Option<i64>,
     }
 
     #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

crates/ipx/src/main.rs

@@ -1531,7 +1531,7 @@ async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
 pub fn due_at(cfg: &config::Config, feed: &config::Feed, state: &db::HttpState, subscribed: bool) -> Option<i64> {
     let last = state.last_checked?;
     let floor = if subscribed { 0 } else { 86_400 };
-    Some(last + due_after(cfg, feed, state.ttl_mins, state.error_since, last).max(floor) as i64)
+    Some(last + due_after(cfg, feed, state.ttl_mins, state.error_since, state.same_since, last).max(floor) as i64)
 }
 
 /// How long the daemon may sleep before a feed is due: until the earliest one, at least 30 s and
@@ -1577,14 +1577,17 @@ pub fn due_after(
     feed: &config::Feed,
     ttl_mins: Option<u64>,
     error_since: Option<i64>,
+    same_since: Option<i64>,
     last_checked: i64,
 ) -> u64 {
-    let mins = match feed.schedule.as_deref().and_then(config::parse_interval) {
-        Some(explicit) => explicit,
-        None => ttl_mins.unwrap_or(0).max(cfg.general.interval()),
+    let explicit = feed.schedule.as_deref().and_then(config::parse_interval);
+    let usual = match explicit {
+        Some(explicit) => explicit * 60,
+        // A schedule someone set is theirs; otherwise a feed long the same is checked less.
+        None => quiet(ttl_mins.unwrap_or(0).max(cfg.general.interval()) * 60, same_since.map(|s| (last_checked - s).max(0) as u64)),
     };
     let failing_for = error_since.map(|since| (last_checked - since).max(0) as u64);
-    backoff(mins * 60, failing_for)
+    backoff(usual, failing_for)
 }
 
 /// A failing feed waits as long as it has been failing, so the wait doubles with each failure
@@ -1592,6 +1595,17 @@ pub fn due_after(
 /// never more than a day. Retried hourly, a feed dead for good cost a request, a warning and scan
 /// time every hour (#99). The first success clears error_since, and with it the backoff; a
 /// refresh someone asks for is not held back by it.
+/// A feed whose body has been the same for a while is checked less often, a twelfth of how long
+/// it has been the same, up to four hours, and on its schedule again once it changes: Patreon and
+/// Substack send the whole feed at every check, ignoring "not modified", 2.6 GB a day (#242).
+fn quiet(usual: u64, same_for: Option<u64>) -> u64 {
+    const CEILING: u64 = 4 * 3600;
+    match same_for {
+        Some(f) => usual.max((f / 12).min(CEILING)),
+        None => usual,
+    }
+}
+
 fn backoff(usual: u64, failing_for: Option<u64>) -> u64 {
     const CEILING: u64 = 86_400;
     match failing_for {
@@ -1721,6 +1735,18 @@ async fn scan_one(
         return Ok(Outcome::Empty);
     }
 
+    // The same body as last time, from a server that ignores "not modified": nothing to read,
+    // as a 304 would have said (#242). Not when something is missing that a read would fill, as
+    // above, nor when someone asked for a full check.
+    let hash = {
+        use sha2::Digest as _;
+        sha2::Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect::<String>()
+    };
+    if !force && state.body_hash.as_deref() == Some(hash.as_str()) && stored.entries > 0 && stored.image.is_some() {
+        ctx.db.touch_feed(id, &feed_cfg.url).await?;
+        return Ok(Outcome::NotModified);
+    }
+
     // A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml"
     // URL; sniffing the body also catches one served from a URL without that extension.
     if feed::is_opml(&bytes) {
@@ -1792,6 +1818,9 @@ async fn scan_one(
         parsed.category.as_deref(),
         parsed.description.as_deref(),
     ).await?;
+    if state.body_hash.as_deref() != Some(hash.as_str()) {
+        ctx.db.set_body(id, &hash).await?;
+    }
     // Asked on a feed's first read, nothing of it stored yet, or when someone forces a refresh:
     // each call costs, and a feed already here without a category can wait for a forced one.
     // The answer becomes the catalogue's category, so a feed is not asked twice.
@@ -2389,6 +2418,40 @@ mod tests {
         assert_eq!(at(Some(1000), false), Some(1000 + 86_400));     // listed, nobody subscribed
     }
 
+    #[test]
+    fn a_feed_long_the_same_is_checked_less_and_a_schedule_set_wins() {
+        let (hour, day) = (3600u64, 86_400u64);
+        assert_eq!(quiet(hour, None), hour);
+        assert_eq!(quiet(hour, Some(6 * hour)), hour, "a few hours the same changes nothing");
+        assert_eq!(quiet(hour, Some(day)), 2 * hour, "a day the same: every two hours");
+        assert_eq!(quiet(hour, Some(30 * day)), 4 * hour, "never past four hours");
+        assert_eq!(quiet(12 * hour, Some(30 * day)), 12 * hour, "never sooner than its schedule");
+        let cfg = config::Config::default();
+        let same = |f: &config::Feed| due_after(&cfg, f, None, None, Some(0), 3 * day as i64);
+        assert_eq!(same(&feed()), 4 * hour);
+        let set = config::Feed { schedule: Some("every 1h".into()), ..feed() };
+        assert_eq!(same(&set), hour, "someone's own schedule wins");
+    }
+
+    #[tokio::test]
+    async fn the_same_body_again_is_not_modified_unless_a_check_is_forced() {
+        let base = empty_or_feed_server().await;
+        let ctx = Arc::new(test_ctx(config::Config::default()).await);
+        let cfg = config::Feed { url: format!("{base}/feed"), ..feed() };
+        assert!(matches!(scan_one(&ctx, "same", &cfg, &db::HttpState::default(), false, None).await, Ok(Outcome::Feed(_))));
+        let state = ctx.db.http_state("same").await.unwrap();
+        let since = state.same_since.expect("when it was first read");
+        assert!(state.body_hash.is_some());
+        // Patreon's way: the whole feed again, no validator honoured (#242).
+        assert!(matches!(scan_one(&ctx, "same", &cfg, &state, false, None).await, Ok(Outcome::NotModified)));
+        assert_eq!(ctx.db.http_state("same").await.unwrap().same_since, Some(since), "still the same since then");
+        assert!(matches!(scan_one(&ctx, "same", &cfg, &state, true, None).await, Ok(Outcome::Feed(_))), "a forced check reads it");
+        // Changed: read, and the same from now.
+        let moved = config::Feed { url: format!("{base}/fixed"), ..feed() };
+        assert!(matches!(scan_one(&ctx, "same", &moved, &state, false, None).await, Ok(Outcome::Feed(_))));
+        assert_ne!(ctx.db.http_state("same").await.unwrap().body_hash, state.body_hash);
+    }
+
     #[test]
     fn a_listed_feed_nobody_subscribes_to_downloads_nothing() {
         let mut f = feed();

crates/ipx/src/web.rs

@@ -770,11 +770,11 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
                 .schedule
                 .as_deref()
                 .and_then(crate::config::parse_interval),
-            every_mins: crate::due_after(&cfg, feed, ttl_mins, None, 0) / 60,
+            every_mins: crate::due_after(&cfg, feed, ttl_mins, None, None, 0) / 60,
             last_checked: s.last_checked,
             next_check: s
                 .last_checked
-                .map(|t| t + crate::due_after(&cfg, feed, ttl_mins, s.error_since, t) as i64),
+                .map(|t| t + crate::due_after(&cfg, feed, ttl_mins, s.error_since, s.same_since, t) as i64),
             failing: s
                 .error_since
                 .filter(|since| crate::db::now() - since >= FLAG_AFTER_SECS)