From 1949f3e4936e732ad9e5cbfc1e5d281a7f19e81d Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 16 Oct 2026 14:02:39 +0000 Subject: [PATCH] Compression across the board: directory answers, uploads, metrics, traces, the page (#253) Measured 2026-10-16, uncompressed until now: the directory's answers (the feed list ipx reads, 603 KB of JSON, ~134 ms of a 159 ms Directory request), ipx's snapshot upload (~100 KB each half hour), both /metrics (4.7 and 0.9 KB every 15 s) and every trace export to Tempo. Already compressed: ipx-web's answers (zstd, #239), the home page and the directory's public pages through Cloudflare (brotli, zstd). - The directory compresses with brotli, zstd or gzip as asked, and unpacks a gzipped request. - ipx sends its snapshot gzipped; the directory is deployed first. - Both /metrics answer gzipped, as Prometheus asks. - opentelemetry-otlp's gzip-http: OTEL_EXPORTER_OTLP_COMPRESSION=gzip compresses the export, set for all three in production's compose file; Caddy's exporter reads the same variable. - web/build.mjs writes .br (quality 11) and .gz (level 9) beside each text file, and Caddy serves them in place (file_server precompressed): app.js 30 KB where zstd on the fly gave 37, app.css 11 where 14. A .br or .gz asked for by name is refused, or /admin.js.br would be the admin page's script without the check that keeps it to admins; in a handle of its own, since a bare respond runs after the handles. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 5 ++++ Cargo.toml | 5 ++-- crates/directory/CHANGELOG.md | 1 + crates/directory/Cargo.toml | 2 ++ crates/directory/src/lib.rs | 45 +++++++++++++++++++++++++++++++++ crates/directory/src/metrics.rs | 4 ++- crates/ipx/CHANGELOG.md | 1 + crates/ipx/Cargo.toml | 1 + crates/ipx/src/directory.rs | 11 +++++++- crates/ipx/src/metrics.rs | 4 ++- tests/ui/app.spec.js | 14 ++++++++++ web/CHANGELOG.md | 1 + web/Caddyfile | 21 ++++++++++++--- web/build.mjs | 10 ++++++++ 14 files changed, 117 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e37f62e..a48fdb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1868,6 +1868,7 @@ dependencies = [ "axum", "chrono", "clap", + "flate2", "futures-util", "ipx-directory", "ipx-feed", @@ -1901,6 +1902,7 @@ dependencies = [ "anyhow", "axum", "clap", + "flate2", "futures-util", "ipx-feed", "mimalloc", @@ -1916,6 +1918,7 @@ dependencies = [ "tokio", "toml", "tower", + "tower-http 0.7.1", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2850,6 +2853,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344" dependencies = [ + "flate2", "http", "httpdate", "opentelemetry", @@ -4690,6 +4694,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ + "async-compression", "bitflags 2.13.1", "bytes", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 0d9190d..83d7843 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,13 +15,14 @@ anyhow = "1.0.104" argon2 = "0.6.0" atom_syndication = "0.12.10" axum = "0.8.9" +flate2 = "1.1.10" chrono = { version = "0.4.45", default-features = false, features = ["std", "clock"] } clap = { version = "4.6.6", features = ["derive"] } futures-util = { version = "0.3.34", default-features = false, features = ["std"] } jsonwebtoken = { version = "11.1.0", default-features = false, features = ["aws_lc_rs"] } librqbit = { version = "9.0.1", default-features = false, features = ["rust-tls", "http-api-client"] } opentelemetry = { version = "0.33", default-features = false, features = ["trace"] } -opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client"] } +opentelemetry-otlp = { version = "0.33", default-features = false, features = ["trace", "http-proto", "reqwest-blocking-client", "gzip-http"] } opentelemetry_sdk = { version = "0.33", default-features = false, features = ["trace"] } opml = "1.1.6" mimalloc = "0.1.52" @@ -36,7 +37,7 @@ sha2 = "0.10.9" tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "sync", "time", "signal", "process"] } toml = "1.1.5" tower = { version = "0.5.3", features = ["util"] } -tower-http = { version = "0.7.1", features = ["fs"] } +tower-http = { version = "0.7.1", features = ["fs", "compression-br", "compression-gzip", "compression-zstd", "decompression-gzip"] } tracing = "0.1.44" tracing-opentelemetry = { version = "0.34", default-features = false } tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index d5ef23a..23acf83 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Answers and pages go out compressed, brotli, zstd or gzip as the caller takes them: the feed list ipx asks for was 603 KB of JSON. A snapshot may come gzipped, `/metrics` answers gzipped, and with `OTEL_EXPORTER_OTLP_COMPRESSION=gzip` traces go to the collector compressed (#253). - With `apple_country` set, Apple Podcasts' top charts for that country (the top 100 and the top 50 of each category) come into the directory every week, beside what servers share, without counting as subscribers; and a warning says when Apple's categories no longer match ipx's (#252). ### Changed diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 9292442..fb6c262 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -26,10 +26,12 @@ serde_json.workspace = true sha2.workspace = true tokio.workspace = true toml.workspace = true +tower-http.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true url.workspace = true [dev-dependencies] +flate2.workspace = true tower.workspace = true diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 982208a..2ecf150 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -268,6 +268,12 @@ pub fn router(dir: Shared) -> Router { .merge(v1) // Anyone may browse, no token, read only (#215). .merge(site::router()) + // Brotli, zstd or gzip as the caller takes them: the feed list went to ipx as 603 KB of + // JSON, most of a Directory page's time (#253). Inside the access log, which records what + // was sent. + .layer(tower_http::compression::CompressionLayer::new().br(true).zstd(true).gzip(true)) + // And takes a request's body gzipped: ipx sends its snapshot so, about 100 KB as JSON. + .layer(tower_http::decompression::RequestDecompressionLayer::new().gzip(true)) .layer(axum::middleware::from_fn(access_log)) .with_state(dir) } @@ -477,6 +483,45 @@ mod tests { (status, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null)) } + #[tokio::test] + async fn a_snapshot_may_come_gzipped() { + use std::io::Write as _; + let (app, _) = app().await; + let json = serde_json::json!([{"url": "https://example.com/feed.xml", "subscribers": 2}]).to_string(); + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(json.as_bytes()).unwrap(); + // As ipx sends it (#253). + let req = axum::http::Request::builder() + .method("POST") + .uri("/v1/snapshot") + .header("authorization", "Bearer secret-rw") + .header("content-type", "application/json") + .header("content-encoding", "gzip") + .body(Body::from(gz.finish().unwrap())) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let got: serde_json::Value = serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap(); + assert_eq!(got["accepted"], 1); + } + + #[tokio::test] + async fn answers_come_compressed_as_the_caller_asks() { + let (app, _) = app().await; + for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] { + // A page, not an empty feed list: under 32 bytes is sent as it is. + let req = axum::http::Request::builder() + .uri("/") + .header("accept-encoding", asked) + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned()); + assert_eq!(got.as_deref(), sent, "asked for {asked} (#253)"); + } + } + async fn get_page(app: &Router, path: &str) -> (u16, String, Option) { let req = axum::http::Request::builder().uri(path).body(Body::empty()).unwrap(); let resp = app.clone().oneshot(req).await.unwrap(); diff --git a/crates/directory/src/metrics.rs b/crates/directory/src/metrics.rs index a31629e..f26d045 100644 --- a/crates/directory/src/metrics.rs +++ b/crates/directory/src/metrics.rs @@ -51,7 +51,9 @@ pub async fn serve(listener: tokio::net::TcpListener) { axum::routing::get(move || async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started)) }), - ); + ) + // Prometheus asks for gzip (#253). + .layer(tower_http::compression::CompressionLayer::new().gzip(true)); if let Err(e) = axum::serve(listener, app).await { tracing::error!(error = %e, "metrics stopped"); } diff --git a/crates/ipx/CHANGELOG.md b/crates/ipx/CHANGELOG.md index 8139189..d8f4891 100644 --- a/crates/ipx/CHANGELOG.md +++ b/crates/ipx/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- With `OTEL_EXPORTER_OTLP_COMPRESSION=gzip`, traces go to the collector compressed; `/metrics` answers gzipped, and a snapshot goes to the directory gzipped (#253). - A Directory request's trace shows the call to the directory whole, with the size of its answer, and the steps that match it against your subscriptions (#250). - The page's report of how long a list took to show after a click (`list_shown`, with the view and filter) is logged under that click's trace (#246). diff --git a/crates/ipx/Cargo.toml b/crates/ipx/Cargo.toml index 6acfabd..e77a2fe 100644 --- a/crates/ipx/Cargo.toml +++ b/crates/ipx/Cargo.toml @@ -9,6 +9,7 @@ ipx-directory.workspace = true ammonia.workspace = true anyhow.workspace = true argon2.workspace = true +flate2.workspace = true atom_syndication.workspace = true axum.workspace = true chrono.workspace = true diff --git a/crates/ipx/src/directory.rs b/crates/ipx/src/directory.rs index 9521950..c97e290 100644 --- a/crates/ipx/src/directory.rs +++ b/crates/ipx/src/directory.rs @@ -65,8 +65,17 @@ async fn call(ctx: &Ctx, method: reqwest::Method, path: &st if let Some(tp) = ipx_directory::otel::traceparent() { req = req.header("traceparent", tp); } + // A snapshot is about 100 KB of JSON, sent gzipped; the directory unpacks it (#253). if let Some(b) = body { - req = req.json(b); + use std::io::Write as _; + let json = serde_json::to_vec(b).map_err(|e| DirError::Unreachable(format!("{e:#}")))?; + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(&json).map_err(|e| DirError::Unreachable(format!("{e:#}")))?; + let packed = gz.finish().map_err(|e| DirError::Unreachable(format!("{e:#}")))?; + req = req + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header(reqwest::header::CONTENT_ENCODING, "gzip") + .body(packed); } let resp = req.send().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?; let status = resp.status(); diff --git a/crates/ipx/src/metrics.rs b/crates/ipx/src/metrics.rs index 397828a..9697f13 100644 --- a/crates/ipx/src/metrics.rs +++ b/crates/ipx/src/metrics.rs @@ -135,7 +135,9 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result { let s = s.clone(); async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(&s)) } }), - ); + ) + // Prometheus asks for gzip, every 15 s (#253). + .layer(tower_http::compression::CompressionLayer::new().gzip(true)); match tokio::net::TcpListener::from_std(listener) { Ok(l) => { if let Err(e) = axum::serve(l, app).await { diff --git a/tests/ui/app.spec.js b/tests/ui/app.spec.js index 9384dd8..8d3dd9e 100644 --- a/tests/ui/app.spec.js +++ b/tests/ui/app.spec.js @@ -2054,3 +2054,17 @@ test('Currently Listening and the Directory show at once once seen, with a spinn } await page.unroute(slow); }); + +test('the page comes compressed, and the compressed copies cannot be asked for by name', async ({ page }) => { + // #253: brotli made at build, served in place of the file. + const js = await page.request.get('/app.js', { headers: { 'Accept-Encoding': 'br, gzip' } }); + expect(js.status()).toBe(200); + expect(js.headers()['content-encoding']).toBe('br'); + expect(js.headers()['content-type']).toContain('javascript'); + const gz = await page.request.get('/app.css', { headers: { 'Accept-Encoding': 'gzip' } }); + expect(gz.headers()['content-encoding']).toBe('gzip'); + // The admin page's script, under its compressed name, would skip the check that keeps it to admins. + for (const p of ['/admin.js.br', '/admin.html.gz', '/app.js.br']) { + expect((await page.request.get(p)).status(), p).toBe(404); + } +}); diff --git a/web/CHANGELOG.md b/web/CHANGELOG.md index ffee19a..07caff7 100644 --- a/web/CHANGELOG.md +++ b/web/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - The page reports how long each list you open takes to show, from the click to the list drawn, for the dashboard (#246). +- The page's files are compressed once when built, brotli at its strongest and gzip, and sent in place of the files: the script 30 KB where it was 37, the stylesheet 11 where it was 14. Traces go to the collector compressed (#253). - The page and the API's answers are sent compressed, zstd or gzip, which the feed list and item lists, over 100 KB each, were not; live updates and audio are not compressed (#239). ## [1.0.2] - 2026-10-15 diff --git a/web/Caddyfile b/web/Caddyfile index 749023f..fcd4982 100644 --- a/web/Caddyfile +++ b/web/Caddyfile @@ -63,6 +63,15 @@ http://{$WEB_BIND::8080} { @compress not path /api/events /media/* encode @compress zstd gzip + # The copies web/build.mjs compressed are served in place of their file, never asked for by + # name: /admin.js.br would be the admin page's script without the check below that keeps it to + # admins (#253). + # A handle of its own: a bare respond runs after the handles, and the last one serves any file. + @packed path *.br *.gz + handle @packed { + respond 404 + } + # Before anything else, so no page or API call sees them: only ipx sets these. @forged not remote_ip {$TUNNEL_PEER:255.255.255.255} request_header @forged -Cf-Access-Authenticated-User-Email @@ -97,7 +106,9 @@ http://{$WEB_BIND::8080} { rewrite / /index.html rewrite /admin /admin.html header Cache-Control no-cache - file_server + file_server { + precompressed br gzip + } } # The sign-in form, except for someone Cloudflare Access vouches for, who is sent to the app. @@ -108,7 +119,9 @@ http://{$WEB_BIND::8080} { } rewrite * /login.html header Cache-Control no-cache - file_server + file_server { + precompressed br gzip + } } # Scripts and the stylesheet are named by a hash of their contents (web/build.mjs), so a browser @@ -120,6 +133,8 @@ http://{$WEB_BIND::8080} { @icons path *.png *.svg *.ico *.woff2 header @icons Cache-Control "public, max-age=86400" handle { - file_server + file_server { + precompressed br gzip + } } } diff --git a/web/build.mjs b/web/build.mjs index 256d32f..0648df0 100644 --- a/web/build.mjs +++ b/web/build.mjs @@ -12,6 +12,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import zlib from 'node:zlib'; import { fileURLToPath } from 'node:url'; import swc from '@swc/core'; import html from '@swc/html'; @@ -113,4 +114,13 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { fs.writeFileSync(path.join(out, name), html); fs.writeFileSync(path.join(out, script), js); } + // Each text file compressed once here, as hard as it goes, brotli and gzip beside it, for Caddy + // to send in its place (#253): smaller than compressing each answer as it goes, and no work then. + for (const f of fs.readdirSync(out).filter(f => /\.(html|js|css|svg|txt)$/.test(f))) { + const body = fs.readFileSync(path.join(out, f)); + fs.writeFileSync(path.join(out, f + '.br'), zlib.brotliCompressSync(body, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, [zlib.constants.BROTLI_PARAM_SIZE_HINT]: body.length }, + })); + fs.writeFileSync(path.join(out, f + '.gz'), zlib.gzipSync(body, { level: 9 })); + } }