Compression across the board: directory answers, uploads, metrics, traces, the page (#253)

Measured 2026-10-16, uncompressed until now: the directory's answers (the feed list ipx reads,
603 KB of JSON, ~134 ms of a 159 ms Directory request), ipx's snapshot upload (~100 KB each half
hour), both /metrics (4.7 and 0.9 KB every 15 s) and every trace export to Tempo. Already
compressed: ipx-web's answers (zstd, #239), the home page and the directory's public pages
through Cloudflare (brotli, zstd).

- The directory compresses with brotli, zstd or gzip as asked, and unpacks a gzipped request.
- ipx sends its snapshot gzipped; the directory is deployed first.
- Both /metrics answer gzipped, as Prometheus asks.
- opentelemetry-otlp's gzip-http: OTEL_EXPORTER_OTLP_COMPRESSION=gzip compresses the export, set
  for all three in production's compose file; Caddy's exporter reads the same variable.
- web/build.mjs writes .br (quality 11) and .gz (level 9) beside each text file, and Caddy serves
  them in place (file_server precompressed): app.js 30 KB where zstd on the fly gave 37, app.css
  11 where 14. A .br or .gz asked for by name is refused, or /admin.js.br would be the admin
  page's script without the check that keeps it to admins; in a handle of its own, since a bare
  respond runs after the handles.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Cargo.lock

@@ -1868,6 +1868,7 @@ dependencies = [
  "axum",
  "chrono",
  "clap",
+ "flate2",
  "futures-util",
  "ipx-directory",
  "ipx-feed",
@@ -1901,6 +1902,7 @@ dependencies = [
  "anyhow",
  "axum",
  "clap",
+ "flate2",
  "futures-util",
  "ipx-feed",
  "mimalloc",
@@ -1916,6 +1918,7 @@ dependencies = [
  "tokio",
  "toml",
  "tower",
+ "tower-http 0.7.1",
  "tracing",
  "tracing-opentelemetry",
  "tracing-subscriber",
@@ -2850,6 +2853,7 @@ version = "0.33.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344"
 dependencies = [
+ "flate2",
  "http",
  "httpdate",
  "opentelemetry",
@@ -4690,6 +4694,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
 dependencies = [
+ "async-compression",
  "bitflags 2.13.1",
  "bytes",
  "futures-core",

Cargo.toml

@@ -15,13 +15,14 @@ anyhow = "1.0.104"
 argon2 = "0.6.0"
 atom_syndication = "0.12.10"
 axum = "0.8.9"
+flate2 = "1.1.10"
 chrono = { version = "0.4.45", default-features = false, features = ["std", "clock"] }
 clap = { version = "4.6.6", features = ["derive"] }
 futures-util = { version = "0.3.34", default-features = false, features = ["std"] }
 jsonwebtoken = { version = "11.1.0", default-features = false, features = ["aws_lc_rs"] }
 librqbit = { version = "9.0.1", default-features = false, features = ["rust-tls", "http-api-client"] }
 opentelemetry = { version = "0.33", default-features = false, features = ["trace"] }
-opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] }
+opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "gzip-http"] }
 opentelemetry_sdk = { version = "0.33", default-features = false, features = ["trace"] }
 opml = "1.1.6"
 mimalloc = "0.1.52"
@@ -36,7 +37,7 @@ sha2 = "0.10.9"
 tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "sync", "time", "signal", "process"] }
 toml = "1.1.5"
 tower = { version = "0.5.3", features = ["util"] }
-tower-http = { version = "0.7.1", features = ["fs"] }
+tower-http = { version = "0.7.1", features = ["fs", "compression-br", "compression-gzip", "compression-zstd", "decompression-gzip"] }
 tracing = "0.1.44"
 tracing-opentelemetry = { version = "0.34", default-features = false }
 tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] }

crates/directory/CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Answers and pages go out compressed, brotli, zstd or gzip as the caller takes them: the feed list ipx asks for was 603 KB of JSON. A snapshot may come gzipped, `/metrics` answers gzipped, and with `OTEL_EXPORTER_OTLP_COMPRESSION=gzip` traces go to the collector compressed (#253).
 - With `apple_country` set, Apple Podcasts' top charts for that country (the top 100 and the top 50 of each category) come into the directory every week, beside what servers share, without counting as subscribers; and a warning says when Apple's categories no longer match ipx's (#252).
 
 ### Changed

crates/directory/Cargo.toml

@@ -26,10 +26,12 @@ serde_json.workspace = true
 sha2.workspace = true
 tokio.workspace = true
 toml.workspace = true
+tower-http.workspace = true
 tracing.workspace = true
 tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
 url.workspace = true
 
 [dev-dependencies]
+flate2.workspace = true
 tower.workspace = true

crates/directory/src/lib.rs

@@ -268,6 +268,12 @@ pub fn router(dir: Shared) -> Router {
         .merge(v1)
         // Anyone may browse, no token, read only (#215).
         .merge(site::router())
+        // Brotli, zstd or gzip as the caller takes them: the feed list went to ipx as 603 KB of
+        // JSON, most of a Directory page's time (#253). Inside the access log, which records what
+        // was sent.
+        .layer(tower_http::compression::CompressionLayer::new().br(true).zstd(true).gzip(true))
+        // And takes a request's body gzipped: ipx sends its snapshot so, about 100 KB as JSON.
+        .layer(tower_http::decompression::RequestDecompressionLayer::new().gzip(true))
         .layer(axum::middleware::from_fn(access_log))
         .with_state(dir)
 }
@@ -477,6 +483,45 @@ mod tests {
         (status, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
     }
 
+    #[tokio::test]
+    async fn a_snapshot_may_come_gzipped() {
+        use std::io::Write as _;
+        let (app, _) = app().await;
+        let json = serde_json::json!([{"url": "https://example.com/feed.xml", "subscribers": 2}]).to_string();
+        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        gz.write_all(json.as_bytes()).unwrap();
+        // As ipx sends it (#253).
+        let req = axum::http::Request::builder()
+            .method("POST")
+            .uri("/v1/snapshot")
+            .header("authorization", "Bearer secret-rw")
+            .header("content-type", "application/json")
+            .header("content-encoding", "gzip")
+            .body(Body::from(gz.finish().unwrap()))
+            .unwrap();
+        let resp = app.clone().oneshot(req).await.unwrap();
+        assert_eq!(resp.status(), 200);
+        let got: serde_json::Value = serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
+        assert_eq!(got["accepted"], 1);
+    }
+
+    #[tokio::test]
+    async fn answers_come_compressed_as_the_caller_asks() {
+        let (app, _) = app().await;
+        for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] {
+            // A page, not an empty feed list: under 32 bytes is sent as it is.
+            let req = axum::http::Request::builder()
+                .uri("/")
+                .header("accept-encoding", asked)
+                .body(Body::empty())
+                .unwrap();
+            let resp = app.clone().oneshot(req).await.unwrap();
+            assert_eq!(resp.status(), 200);
+            let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned());
+            assert_eq!(got.as_deref(), sent, "asked for {asked} (#253)");
+        }
+    }
+
     async fn get_page(app: &Router, path: &str) -> (u16, String, Option<String>) {
         let req = axum::http::Request::builder().uri(path).body(Body::empty()).unwrap();
         let resp = app.clone().oneshot(req).await.unwrap();

crates/directory/src/metrics.rs

@@ -51,7 +51,9 @@ pub async fn serve(listener: tokio::net::TcpListener) {
         axum::routing::get(move || async move {
             ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
         }),
-    );
+    )
+    // Prometheus asks for gzip (#253).
+    .layer(tower_http::compression::CompressionLayer::new().gzip(true));
     if let Err(e) = axum::serve(listener, app).await {
         tracing::error!(error = %e, "metrics stopped");
     }

crates/ipx/CHANGELOG.md

@@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- With `OTEL_EXPORTER_OTLP_COMPRESSION=gzip`, traces go to the collector compressed; `/metrics` answers gzipped, and a snapshot goes to the directory gzipped (#253).
 - A Directory request's trace shows the call to the directory whole, with the size of its answer, and the steps that match it against your subscriptions (#250).
 - The page's report of how long a list took to show after a click (`list_shown`, with the view and filter) is logged under that click's trace (#246).
 

crates/ipx/Cargo.toml

@@ -9,6 +9,7 @@ ipx-directory.workspace = true
 ammonia.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
+flate2.workspace = true
 atom_syndication.workspace = true
 axum.workspace = true
 chrono.workspace = true

crates/ipx/src/directory.rs

@@ -65,8 +65,17 @@ async fn call<T: DeserializeOwned>(ctx: &Ctx, method: reqwest::Method, path: &st
     if let Some(tp) = ipx_directory::otel::traceparent() {
         req = req.header("traceparent", tp);
     }
+    // A snapshot is about 100 KB of JSON, sent gzipped; the directory unpacks it (#253).
     if let Some(b) = body {
-        req = req.json(b);
+        use std::io::Write as _;
+        let json = serde_json::to_vec(b).map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
+        let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
+        gz.write_all(&json).map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
+        let packed = gz.finish().map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
+        req = req
+            .header(reqwest::header::CONTENT_TYPE, "application/json")
+            .header(reqwest::header::CONTENT_ENCODING, "gzip")
+            .body(packed);
     }
     let resp = req.send().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
     let status = resp.status();

crates/ipx/src/metrics.rs

@@ -135,7 +135,9 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
                     let s = s.clone();
                     async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(&s)) }
                 }),
-            );
+            )
+            // Prometheus asks for gzip, every 15 s (#253).
+            .layer(tower_http::compression::CompressionLayer::new().gzip(true));
             match tokio::net::TcpListener::from_std(listener) {
                 Ok(l) => {
                     if let Err(e) = axum::serve(l, app).await {

tests/ui/app.spec.js

@@ -2054,3 +2054,17 @@ test('Currently Listening and the Directory show at once once seen, with a spinn
   }
   await page.unroute(slow);
 });
+
+test('the page comes compressed, and the compressed copies cannot be asked for by name', async ({ page }) => {
+  // #253: brotli made at build, served in place of the file.
+  const js = await page.request.get('/app.js', { headers: { 'Accept-Encoding': 'br, gzip' } });
+  expect(js.status()).toBe(200);
+  expect(js.headers()['content-encoding']).toBe('br');
+  expect(js.headers()['content-type']).toContain('javascript');
+  const gz = await page.request.get('/app.css', { headers: { 'Accept-Encoding': 'gzip' } });
+  expect(gz.headers()['content-encoding']).toBe('gzip');
+  // The admin page's script, under its compressed name, would skip the check that keeps it to admins.
+  for (const p of ['/admin.js.br', '/admin.html.gz', '/app.js.br']) {
+    expect((await page.request.get(p)).status(), p).toBe(404);
+  }
+});

web/CHANGELOG.md

@@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - The page reports how long each list you open takes to show, from the click to the list drawn, for the dashboard (#246).
+- The page's files are compressed once when built, brotli at its strongest and gzip, and sent in place of the files: the script 30 KB where it was 37, the stylesheet 11 where it was 14. Traces go to the collector compressed (#253).
 - The page and the API's answers are sent compressed, zstd or gzip, which the feed list and item lists, over 100 KB each, were not; live updates and audio are not compressed (#239).
 
 ## [1.0.2] - 2026-10-15

web/Caddyfile

@@ -63,6 +63,15 @@ http://{$WEB_BIND::8080} {
 	@compress not path /api/events /media/*
 	encode @compress zstd gzip
 
+	# The copies web/build.mjs compressed are served in place of their file, never asked for by
+	# name: /admin.js.br would be the admin page's script without the check below that keeps it to
+	# admins (#253).
+	# A handle of its own: a bare respond runs after the handles, and the last one serves any file.
+	@packed path *.br *.gz
+	handle @packed {
+		respond 404
+	}
+
 	# Before anything else, so no page or API call sees them: only ipx sets these.
 	@forged not remote_ip {$TUNNEL_PEER:255.255.255.255}
 	request_header @forged -Cf-Access-Authenticated-User-Email
@@ -97,7 +106,9 @@ http://{$WEB_BIND::8080} {
 		rewrite / /index.html
 		rewrite /admin /admin.html
 		header Cache-Control no-cache
-		file_server
+		file_server {
+			precompressed br gzip
+		}
 	}
 
 	# The sign-in form, except for someone Cloudflare Access vouches for, who is sent to the app.
@@ -108,7 +119,9 @@ http://{$WEB_BIND::8080} {
 		}
 		rewrite * /login.html
 		header Cache-Control no-cache
-		file_server
+		file_server {
+			precompressed br gzip
+		}
 	}
 
 	# Scripts and the stylesheet are named by a hash of their contents (web/build.mjs), so a browser
@@ -120,6 +133,8 @@ http://{$WEB_BIND::8080} {
 	@icons path *.png *.svg *.ico *.woff2
 	header @icons Cache-Control "public, max-age=86400"
 	handle {
-		file_server
+		file_server {
+			precompressed br gzip
+		}
 	}
 }

web/build.mjs

@@ -12,6 +12,7 @@
 import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
+import zlib from 'node:zlib';
 import { fileURLToPath } from 'node:url';
 import swc from '@swc/core';
 import html from '@swc/html';
@@ -113,4 +114,13 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
     fs.writeFileSync(path.join(out, name), html);
     fs.writeFileSync(path.join(out, script), js);
   }
+  // Each text file compressed once here, as hard as it goes, brotli and gzip beside it, for Caddy
+  // to send in its place (#253): smaller than compressing each answer as it goes, and no work then.
+  for (const f of fs.readdirSync(out).filter(f => /\.(html|js|css|svg|txt)$/.test(f))) {
+    const body = fs.readFileSync(path.join(out, f));
+    fs.writeFileSync(path.join(out, f + '.br'), zlib.brotliCompressSync(body, {
+      params: { [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, [zlib.constants.BROTLI_PARAM_SIZE_HINT]: body.length },
+    }));
+    fs.writeFileSync(path.join(out, f + '.gz'), zlib.gzipSync(body, { level: 9 }));
+  }
 }